fix(cursor): align extractor raw params and handle fractional requestsCosts

jbsmith7741 · jbsmith7741 · commit 3013bbb46fb6 · 2026-07-28T11:30:46.000-06:00
Signed-off-by: Joshua Smith &lt;jbsmith7741@gmail.com&gt;
diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go
@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/apache/incubator-devlake/core/config"
+	"github.com/apache/incubator-devlake/core/models/common"
+	"github.com/apache/incubator-devlake/core/runner"
+	"github.com/apache/incubator-devlake/helpers/e2ehelper"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/cursor/impl"
+	"github.com/apache/incubator-devlake/plugins/cursor/models"
+	"github.com/apache/incubator-devlake/plugins/cursor/tasks"
+)
+
+func TestCursorExtractorsDataFlow(t *testing.T) {
+	cfg := config.GetConfig()
+	dbUrl := cfg.GetString("E2E_DB_URL")
+	if dbUrl == "" {
+		t.Skip("skipping e2e test: E2E_DB_URL is not set")
+	}
+	if err := runner.CheckDbConnection(dbUrl, 10*time.Second); err != nil {
+		t.Skipf("skipping e2e test: cannot connect to E2E_DB_URL: %v", err)
+	}
+
+	var cursorPlugin impl.Cursor
+	dataflowTester := e2ehelper.NewDataFlowTester(t, "cursor", cursorPlugin)
+
+	taskData := &tasks.CursorTaskData{
+		Options: &tasks.CursorOptions{
+			ConnectionId: 1,
+			ScopeId:      "team",
+		},
+		Connection: &models.CursorConnection{
+			CursorConn: models.CursorConn{
+				RestConnection: helper.RestConnection{Endpoint: models.DefaultEndpoint},
+			},
+		},
+	}
+
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_usage_events.csv", "_raw_cursor_usage_events")
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_members.csv", "_raw_cursor_members")
+	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_user_spend.csv", "_raw_cursor_user_spend")
+
+	dataflowTester.FlushTabler(&models.CursorUsageEvent{})
+	dataflowTester.FlushTabler(&models.CursorMember{})
+	dataflowTester.FlushTabler(&models.CursorUserSpend{})
+
+	dataflowTester.Subtask(tasks.ExtractMembersMeta, taskData)
+	dataflowTester.Subtask(tasks.ExtractUsageEventsMeta, taskData)
+	dataflowTester.Subtask(tasks.ExtractUserSpendMeta, taskData)
+
+	dataflowTester.VerifyTableWithOptions(&models.CursorUsageEvent{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/_tool_cursor_usage_events.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+
+	dataflowTester.VerifyTableWithOptions(&models.CursorMember{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/_tool_cursor_members.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+
+	dataflowTester.VerifyTableWithOptions(&models.CursorUserSpend{}, e2ehelper.TableOptions{
+		CSVRelPath:  "./snapshot_tables/_tool_cursor_user_spend.csv",
+		IgnoreTypes: []interface{}{common.NoPKModel{}},
+	})
+}
diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""name"":""Example User"",""email"":""user@example.com"",""id"":""user_example123"",""role"":""member"",""isRemoved"":false}",https://api.cursor.com/teams/members,null,2026-07-09 17:14:19.000
diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""timestamp"":""1783617259310"",""model"":""composer-2.5-fast"",""kind"":""Usage-based"",""maxMode"":false,""requestsCosts"":2,""isTokenBasedCall"":false,""tokenUsage"":{""inputTokens"":5932,""outputTokens"":4761,""cacheWriteTokens"":0,""cacheReadTokens"":495701,""totalCents"":33.70615},""userEmail"":""user@example.com"",""isChargeable"":true,""serviceAccountId"":""null"",""isHeadless"":false,""chargedCents"":8,""conversationId"":""846952e0-0221-473b-bb1d-e5d016f071b1""}",https://api.cursor.com/teams/filtered-usage-events,null,2026-07-09 17:14:19.000
diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv
@@ -0,0 +1,2 @@
+id,params,data,url,input,created_at
+1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""subscriptionCycleStart"":1781472188000,""collectedAt"":""2026-07-10T17:31:24.141244Z"",""member"":{""userId"":""user_example123"",""spendCents"":3224,""includedSpendCents"":1984,""fastPremiumRequests"":496,""name"":""Example User"",""email"":""user@example.com"",""profilePictureUrl"":null,""role"":""member"",""monthlyLimitDollars"":null,""hardLimitOverrideDollars"":0}}",https://api.cursor.com/teams/spend,null,2026-07-10 17:31:24.000
diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv
@@ -0,0 +1,2 @@
+connection_id,scope_id,user_id,email,name,role,is_removed
+1,team,user_example123,user@example.com,Example User,member,0
diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv
@@ -0,0 +1,2 @@
+connection_id,scope_id,event_id,event_time,user_email,model,kind,conversation_id,charged_cents,requests_costs,is_token_based_call,is_chargeable,max_mode,is_headless,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,total_cents,cursor_token_fee,hosting_type,service_account_id
+1,team,2fe8f95cb983e68c7e944264bf1cc976c4940e1d3045f8fd5715fb66466d6eb7,2026-07-09T17:14:19.310+00:00,user@example.com,composer-2.5-fast,Usage-based,846952e0-0221-473b-bb1d-e5d016f071b1,8,2,0,1,0,0,5932,4761,495701,0,33.70615,0,,
diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv
@@ -0,0 +1,2 @@
+connection_id,scope_id,user_id,billing_cycle_start,collected_at,email,name,role,spend_cents,included_spend_cents,fast_premium_requests,monthly_limit_dollars,hard_limit_override_dollars
+1,team,user_example123,2026-06-14T21:23:08.000+00:00,2026-07-10T17:31:24.141+00:00,user@example.com,Example User,member,3224,1984,496,0,0
diff --git a/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go b/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go
@@ -0,0 +1,39 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+)
+
+type changeCursorRequestsCostsToFloat struct{}
+
+func (script *changeCursorRequestsCostsToFloat) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if !db.HasTable("_tool_cursor_usage_events") {
+		return nil
+	}
+	return db.Exec("ALTER TABLE _tool_cursor_usage_events MODIFY COLUMN requests_costs DOUBLE")
+}
+
+func (*changeCursorRequestsCostsToFloat) Version() uint64 { return 20260710120000 }
+
+func (*changeCursorRequestsCostsToFloat) Name() string {
+	return "cursor change requests_costs column to double"
+}
diff --git a/backend/plugins/cursor/models/migrationscripts/register.go b/backend/plugins/cursor/models/migrationscripts/register.go
@@ -23,5 +23,6 @@ import "github.com/apache/incubator-devlake/core/plugin"
 func All() []plugin.MigrationScript {
 	return []plugin.MigrationScript{
 		new(addCursorInitialTables),
+		new(changeCursorRequestsCostsToFloat),
 	}
 }
diff --git a/backend/plugins/cursor/models/usage_event.go b/backend/plugins/cursor/models/usage_event.go
@@ -35,7 +35,7 @@ type CursorUsageEvent struct {
 	Kind             string  `json:"kind" gorm:"type:varchar(100)"`
 	ConversationId   string  `json:"conversationId" gorm:"type:varchar(64);index"`
 	ChargedCents     float64 `json:"chargedCents"`
-	RequestsCosts    int     `json:"requestsCosts"`
+	RequestsCosts    float64 `json:"requestsCosts"`
 	IsTokenBasedCall bool    `json:"isTokenBasedCall"`
 	IsChargeable     bool    `json:"isChargeable"`
 	MaxMode          bool    `json:"maxMode"`
diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/plugins/cursor/models"
 )
 
 const (
@@ -50,6 +51,19 @@ func (p cursorRawParams) GetParams() any {
 	return p
 }
 
+func rawParamsFromTaskData(data *CursorTaskData) cursorRawParams {
+	endpoint := models.DefaultEndpoint
+	if data.Connection != nil {
+		data.Connection.Normalize()
+		endpoint = data.Connection.Endpoint
+	}
+	return cursorRawParams{
+		ConnectionId: data.Options.ConnectionId,
+		ScopeId:      data.Options.ScopeId,
+		Endpoint:     endpoint,
+	}
+}
+
 type cursorTimeRangeInput struct {
 	StartDateMs int64 `json:"startDateMs"`
 	EndDateMs   int64 `json:"endDateMs"`
@@ -176,8 +190,8 @@ func parseEventTimestampMs(raw string) (time.Time, errors.Error) {
 	return time.UnixMilli(ms).UTC(), nil
 }
 
-func computeEventId(timestamp, userEmail, conversationId, model string, chargedCents float64, requestsCosts int) string {
-	payload := fmt.Sprintf("%s|%s|%s|%s|%v|%d", timestamp, userEmail, conversationId, model, chargedCents, requestsCosts)
+func computeEventId(timestamp, userEmail, conversationId, model string, chargedCents, requestsCosts float64) string {
+	payload := fmt.Sprintf("%s|%s|%s|%s|%v|%v", timestamp, userEmail, conversationId, model, chargedCents, requestsCosts)
 	sum := sha256.Sum256([]byte(payload))
 	return hex.EncodeToString(sum[:])
 }
diff --git a/backend/plugins/cursor/tasks/collector_utils_test.go b/backend/plugins/cursor/tasks/collector_utils_test.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+	"encoding/json"
+	"testing"
+
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/cursor/models"
+)
+
+func TestRawParamsFromTaskDataIncludesEndpoint(t *testing.T) {
+	taskData := &CursorTaskData{
+		Options: &CursorOptions{
+			ConnectionId: 1,
+			ScopeId:      "team",
+		},
+		Connection: &models.CursorConnection{
+			CursorConn: models.CursorConn{
+				RestConnection: helper.RestConnection{Endpoint: models.DefaultEndpoint},
+			},
+		},
+	}
+
+	params := rawParamsFromTaskData(taskData)
+	encoded, err := json.Marshal(params.GetParams())
+	if err != nil {
+		t.Fatalf("marshal params: %v", err)
+	}
+
+	expected := `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}`
+	if string(encoded) != expected {
+		t.Fatalf("raw params mismatch:\n got: %s\nwant: %s", encoded, expected)
+	}
+}
diff --git a/backend/plugins/cursor/tasks/members_collector.go b/backend/plugins/cursor/tasks/members_collector.go
@@ -29,22 +29,15 @@ func CollectMembers(taskCtx plugin.SubTaskContext) errors.Error {
 	if !ok {
 		return errors.Default.New("task data is not CursorTaskData")
 	}
-	connection := data.Connection
-	connection.Normalize()
-
-	apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection)
+	apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection)
 	if err != nil {
 		return err
 	}
 
 	rawArgs := helper.RawDataSubTaskArgs{
-		Ctx:   taskCtx,
-		Table: rawMembersTable,
-		Options: cursorRawParams{
-			ConnectionId: data.Options.ConnectionId,
-			ScopeId:      data.Options.ScopeId,
-			Endpoint:     connection.Endpoint,
-		},
+		Ctx:     taskCtx,
+		Table:   rawMembersTable,
+		Options: rawParamsFromTaskData(data),
 	}
 
 	collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
diff --git a/backend/plugins/cursor/tasks/members_extractor.go b/backend/plugins/cursor/tasks/members_extractor.go
@@ -46,10 +46,7 @@ func ExtractMembers(taskCtx plugin.SubTaskContext) errors.Error {
 		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
 			Ctx:   taskCtx,
 			Table: rawMembersTable,
-			Options: cursorRawParams{
-				ConnectionId: data.Options.ConnectionId,
-				ScopeId:      data.Options.ScopeId,
-			},
+			Options: rawParamsFromTaskData(data),
 		},
 		Extract: func(row *helper.RawData) ([]interface{}, errors.Error) {
 			var record memberRecord
diff --git a/backend/plugins/cursor/tasks/usage_events_collector.go b/backend/plugins/cursor/tasks/usage_events_collector.go
@@ -72,22 +72,15 @@ func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error {
 	if !ok {
 		return errors.Default.New("task data is not CursorTaskData")
 	}
-	connection := data.Connection
-	connection.Normalize()
-
-	apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection)
+	apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection)
 	if err != nil {
 		return err
 	}
 
 	rawArgs := helper.RawDataSubTaskArgs{
-		Ctx:   taskCtx,
-		Table: rawUsageEventsTable,
-		Options: cursorRawParams{
-			ConnectionId: data.Options.ConnectionId,
-			ScopeId:      data.Options.ScopeId,
-			Endpoint:     connection.Endpoint,
-		},
+		Ctx:     taskCtx,
+		Table:   rawUsageEventsTable,
+		Options: rawParamsFromTaskData(data),
 	}
 
 	collector, err := helper.NewStatefulApiCollector(rawArgs)
diff --git a/backend/plugins/cursor/tasks/usage_events_extractor.go b/backend/plugins/cursor/tasks/usage_events_extractor.go
@@ -32,7 +32,7 @@ type usageEventRecord struct {
 	Model            string           `json:"model"`
 	Kind             string           `json:"kind"`
 	MaxMode          bool             `json:"maxMode"`
-	RequestsCosts    int              `json:"requestsCosts"`
+	RequestsCosts    float64          `json:"requestsCosts"`
 	IsTokenBasedCall bool             `json:"isTokenBasedCall"`
 	TokenUsage       *usageTokenUsage `json:"tokenUsage"`
 	UserEmail        string           `json:"userEmail"`
@@ -64,10 +64,7 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error {
 		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
 			Ctx:   taskCtx,
 			Table: rawUsageEventsTable,
-			Options: cursorRawParams{
-				ConnectionId: data.Options.ConnectionId,
-				ScopeId:      data.Options.ScopeId,
-			},
+			Options: rawParamsFromTaskData(data),
 		},
 		Extract: func(row *helper.RawData) ([]interface{}, errors.Error) {
 			var record usageEventRecord
diff --git a/backend/plugins/cursor/tasks/usage_events_extractor_test.go b/backend/plugins/cursor/tasks/usage_events_extractor_test.go
diff --git a/backend/plugins/cursor/tasks/user_spend_collector.go b/backend/plugins/cursor/tasks/user_spend_collector.go
diff --git a/backend/plugins/cursor/tasks/user_spend_extractor.go b/backend/plugins/cursor/tasks/user_spend_extractor.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+id,params,data,url,input,created_at`
	`2`	`+1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""name"":""Example User"",""email"":""user@example.com"",""id"":""user_example123"",""role"":""member"",""isRemoved"":false}",https://api.cursor.com/teams/members,null,2026-07-09 17:14:19.000`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+id,params,data,url,input,created_at`
	`2`	+1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""timestamp"":""1783617259310"",""model"":""composer-2.5-fast"",""kind"":""Usage-based"",""maxMode"":false,""requestsCosts"":2,""isTokenBasedCall"":false,""tokenUsage"":{""inputTokens"":5932,""outputTokens"":4761,""cacheWriteTokens"":0,""cacheReadTokens"":495701,""totalCents"":33.70615},""userEmail"":""user@example.com"",""isChargeable"":true,""serviceAccountId"":""null"",""isHeadless"":false,""chargedCents"":8,""conversationId"":""846952e0-0221-473b-bb1d-e5d016f071b1""}",https://api.cursor.com/teams/filtered-usage-events,null,2026-07-09 17:14:19.000
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+connection_id,scope_id,user_id,email,name,role,is_removed`
	`2`	`+1,team,user_example123,user@example.com,Example User,member,0`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+connection_id,scope_id,event_id,event_time,user_email,model,kind,conversation_id,charged_cents,requests_costs,is_token_based_call,is_chargeable,max_mode,is_headless,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,total_cents,cursor_token_fee,hosting_type,service_account_id`
	`2`	`+1,team,2fe8f95cb983e68c7e944264bf1cc976c4940e1d3045f8fd5715fb66466d6eb7,2026-07-09T17:14:19.310+00:00,user@example.com,composer-2.5-fast,Usage-based,846952e0-0221-473b-bb1d-e5d016f071b1,8,2,0,1,0,0,5932,4761,495701,0,33.70615,0,,`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+connection_id,scope_id,user_id,billing_cycle_start,collected_at,email,name,role,spend_cents,included_spend_cents,fast_premium_requests,monthly_limit_dollars,hard_limit_override_dollars`
	`2`	`+1,team,user_example123,2026-06-14T21:23:08.000+00:00,2026-07-10T17:31:24.141+00:00,user@example.com,Example User,member,3224,1984,496,0,0`
Original file line number	Diff line number	Diff line change
`@@ -23,5 +23,6 @@ import "github.com/apache/incubator-devlake/core/plugin"`
`23`	`23`	`func All() []plugin.MigrationScript {`
`24`	`24`	`return []plugin.MigrationScript{`
`25`	`25`	`new(addCursorInitialTables),`
	`26`	`+ new(changeCursorRequestsCostsToFloat),`
`26`	`27`	`}`
`27`	`28`	`}`