Skip to content

Commit 8f6b832

Browse files
fix(customize): skip orphaned _raw_data_id rows instead of aborting extraction
ExtractCustomizedFields used a LEFT JOIN to the raw table, so a domain row whose _raw_data_id has no matching raw record yields NULL data. That hit the type switch default branch, which did 'return nil' and exited the whole subtask, silently leaving every later row in the (unordered) cursor unprocessed while still reporting success. Change the default branch to 'continue' so an orphaned row is skipped and extraction proceeds with the remaining rows. Closes #8945
1 parent 57b2641 commit 8f6b832

2 files changed

Lines changed: 86 additions & 1 deletion

File tree

backend/plugins/customize/tasks/customized_fields_extractor.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,9 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal, table, rawTable, ra
142142
}
143143
}
144144
default:
145-
return nil
145+
// orphaned/NULL raw data (LEFT JOIN found no matching raw record): skip this
146+
// row and keep scanning, instead of aborting the whole subtask.
147+
continue
146148
}
147149

148150
if len(updates) > 0 {
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"context"
22+
"testing"
23+
24+
"github.com/apache/incubator-devlake/core/dal"
25+
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
26+
"github.com/stretchr/testify/assert"
27+
"github.com/stretchr/testify/mock"
28+
)
29+
30+
// TestExtractCustomizedFieldsSkipsOrphanedRows is a regression test for the case where a
31+
// domain-layer row's `_raw_data_id` has no matching raw record (LEFT JOIN yields NULL data).
32+
// Such a row must be skipped and extraction must continue with the remaining rows, instead of
33+
// aborting the whole scan on the first orphaned row.
34+
func TestExtractCustomizedFieldsSkipsOrphanedRows(t *testing.T) {
35+
mockRows := new(mockdal.Rows)
36+
mockRows.On("Next").Return(true).Times(2)
37+
mockRows.On("Next").Return(false).Once()
38+
mockRows.On("Close").Return(nil).Once()
39+
40+
mockDal := new(mockdal.Dal)
41+
mockDal.On("GetColumns", mock.Anything, mock.Anything).Return([]dal.ColumnMeta{}, nil)
42+
mockDal.On("Cursor", mock.Anything).Return(mockRows, nil).Once()
43+
44+
// Row 1 is orphaned: no matching raw record, so `data` is NULL (nil). It must be skipped.
45+
mockDal.On("Fetch", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
46+
dst := args.Get(1).(*map[string]interface{})
47+
*dst = map[string]interface{}{
48+
"id": "row-orphaned",
49+
"_raw_data_id": int64(1),
50+
"data": nil,
51+
}
52+
}).Return(nil).Once()
53+
// Row 2 has valid raw data and matches the filter; it must still be processed.
54+
mockDal.On("Fetch", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
55+
dst := args.Get(1).(*map[string]interface{})
56+
*dst = map[string]interface{}{
57+
"id": "row-valid",
58+
"_raw_data_id": int64(2),
59+
"data": `{"name":"hello"}`,
60+
}
61+
}).Return(nil).Once()
62+
63+
execCalls := 0
64+
mockDal.On("Exec", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
65+
execCalls++
66+
}).Return(nil)
67+
68+
err := extractCustomizedFields(
69+
context.Background(),
70+
mockDal,
71+
"boards", // non-issues table -> simple column update path
72+
"_raw_jira_api_issues",
73+
`{"ConnectionId":1`,
74+
map[string]string{"x_test": "name"},
75+
)
76+
77+
assert.Nil(t, err)
78+
// The valid row after the orphaned one must have been updated exactly once.
79+
// Before the fix, the orphaned row hit `default: return nil` and aborted the scan,
80+
// so Exec was never called.
81+
assert.Equal(t, 1, execCalls)
82+
mockDal.AssertExpectations(t)
83+
}

0 commit comments

Comments
 (0)