Skip to content

Commit b34d761

Browse files
DoDiODevDoDiODev
authored
fix(sonarqube): support long issue code block components (apache#8986)
Signed-off-by: DoDiODev <DoDiDev@proton.me>
1 parent 060af3a commit b34d761

8 files changed

Lines changed: 336 additions & 2 deletions

File tree

‎backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import "github.com/apache/incubator-devlake/core/models/domainlayer"
2222
type CqIssueCodeBlock struct {
2323
domainlayer.DomainEntity
2424
IssueKey string `json:"key" gorm:"index"`
25-
Component string `gorm:"index"`
25+
Component string `gorm:"type:text"`
2626
StartLine int
2727
EndLine int
2828
StartOffset int

‎backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/core/plugin"
24+
)
25+
26+
var _ plugin.MigrationScript = (*changeCqIssueCodeBlocksComponentToText)(nil)
27+
28+
type changeCqIssueCodeBlocksComponentToText struct{}
29+
30+
func (script *changeCqIssueCodeBlocksComponentToText) Up(basicRes context.BasicRes) errors.Error {
31+
db := basicRes.GetDal()
32+
if err := db.DropIndexes("cq_issue_code_blocks", "idx_cq_issue_code_blocks_component"); err != nil {
33+
return err
34+
}
35+
return db.ModifyColumnType("cq_issue_code_blocks", "component", "text")
36+
}
37+
38+
func (*changeCqIssueCodeBlocksComponentToText) Version() uint64 {
39+
return 20260701000000
40+
}
41+
42+
func (*changeCqIssueCodeBlocksComponentToText) Name() string {
43+
return "change cq_issue_code_blocks.component type to text"
44+
}

‎backend/core/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ func All() []plugin.MigrationScript {
145145
new(modifyCicdDeploymentsToText),
146146
new(increaseCqIssuesProjectKeyLength),
147147
new(addAuthSessions),
148+
new(changeCqIssueCodeBlocksComponentToText),
148149
new(addCqProjectMetricsHistory),
149150
}
150151
}

‎backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package e2e
19+
20+
import (
21+
"strings"
22+
"testing"
23+
"time"
24+
25+
"github.com/apache/incubator-devlake/core/models/common"
26+
"github.com/apache/incubator-devlake/core/models/domainlayer"
27+
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
28+
coremigrations "github.com/apache/incubator-devlake/core/models/migrationscripts"
29+
"github.com/apache/incubator-devlake/core/plugin"
30+
"github.com/apache/incubator-devlake/helpers/e2ehelper"
31+
implcontext "github.com/apache/incubator-devlake/impls/context"
32+
"github.com/apache/incubator-devlake/plugins/sonarqube/impl"
33+
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
34+
sonarqubemigrations "github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts"
35+
"github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
36+
"github.com/stretchr/testify/require"
37+
)
38+
39+
type sonarqubeIssueCodeBlockBeforeText struct {
40+
ConnectionId uint64 `gorm:"primaryKey"`
41+
Id string `gorm:"primaryKey"`
42+
IssueKey string `gorm:"index"`
43+
Component string `gorm:"index;type:varchar(500)"`
44+
StartLine int
45+
EndLine int
46+
StartOffset int
47+
EndOffset int
48+
Msg string
49+
common.NoPKModel
50+
}
51+
52+
func (sonarqubeIssueCodeBlockBeforeText) TableName() string {
53+
return "_tool_sonarqube_issue_code_blocks"
54+
}
55+
56+
type cqIssueCodeBlockBeforeText struct {
57+
domainlayer.DomainEntity
58+
IssueKey string `json:"key" gorm:"index"`
59+
Component string `gorm:"index"`
60+
StartLine int
61+
EndLine int
62+
StartOffset int
63+
EndOffset int
64+
Msg string
65+
}
66+
67+
func (cqIssueCodeBlockBeforeText) TableName() string {
68+
return "cq_issue_code_blocks"
69+
}
70+
71+
func TestSonarqubeIssueCodeBlockLongComponent(t *testing.T) {
72+
var sonarqube impl.Sonarqube
73+
dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube)
74+
dataflowTester.FlushTabler(&models.SonarqubeIssue{})
75+
require.NoError(t, dataflowTester.Db.Migrator().DropTable(
76+
&sonarqubeIssueCodeBlockBeforeText{},
77+
&cqIssueCodeBlockBeforeText{},
78+
))
79+
require.NoError(t, dataflowTester.Db.AutoMigrate(
80+
&sonarqubeIssueCodeBlockBeforeText{},
81+
&cqIssueCodeBlockBeforeText{},
82+
))
83+
84+
existingComponent := "existing:component"
85+
require.NoError(t, dataflowTester.Db.Create(&sonarqubeIssueCodeBlockBeforeText{
86+
ConnectionId: 1,
87+
Id: "existing-tool-block",
88+
IssueKey: "existing-issue",
89+
Component: existingComponent,
90+
}).Error)
91+
require.NoError(t, dataflowTester.Db.Create(&cqIssueCodeBlockBeforeText{
92+
DomainEntity: domainlayer.DomainEntity{Id: "existing-domain-block"},
93+
IssueKey: "existing-domain-issue",
94+
Component: existingComponent,
95+
}).Error)
96+
97+
basicRes := implcontext.NewDefaultBasicRes(dataflowTester.Cfg, dataflowTester.Log, dataflowTester.Dal)
98+
runMigration(t, coremigrations.All(), "change cq_issue_code_blocks.component type to text", basicRes)
99+
runMigration(t, sonarqubemigrations.All(), "change _tool_sonarqube_issue_code_blocks.component type to text", basicRes)
100+
assertTextColumnWithoutIndex(t, dataflowTester, "cq_issue_code_blocks")
101+
assertTextColumnWithoutIndex(t, dataflowTester, "_tool_sonarqube_issue_code_blocks")
102+
103+
var migratedToolBlock sonarqubeIssueCodeBlockBeforeText
104+
require.NoError(t, dataflowTester.Db.First(&migratedToolBlock, "id = ?", "existing-tool-block").Error)
105+
require.Equal(t, existingComponent, migratedToolBlock.Component)
106+
var migratedDomainBlock cqIssueCodeBlockBeforeText
107+
require.NoError(t, dataflowTester.Db.First(&migratedDomainBlock, "id = ?", "existing-domain-block").Error)
108+
require.Equal(t, existingComponent, migratedDomainBlock.Component)
109+
require.NoError(t, dataflowTester.Db.Delete(&migratedToolBlock).Error)
110+
require.NoError(t, dataflowTester.Db.Delete(&migratedDomainBlock).Error)
111+
112+
longComponent256 := "project:" + strings.Repeat("a", 256)
113+
longComponent500 := "project:" + strings.Repeat("b", 500)
114+
require.Greater(t, len(longComponent256), 256)
115+
require.Greater(t, len(longComponent500), 500)
116+
117+
issueKey := "TEST-LONG-COMPONENT-ISSUE"
118+
projectKey := "test-long-component-project"
119+
result := dataflowTester.Db.Create(&models.SonarqubeIssue{
120+
ConnectionId: 1,
121+
IssueKey: issueKey,
122+
ProjectKey: projectKey,
123+
Component: longComponent500,
124+
Rule: "java:S3776",
125+
Severity: "CRITICAL",
126+
})
127+
require.NoError(t, result.Error)
128+
129+
codeBlocks := []*models.SonarqubeIssueCodeBlock{
130+
{
131+
ConnectionId: 1,
132+
Id: "test-long-component-block-256",
133+
IssueKey: issueKey,
134+
Component: longComponent256,
135+
Msg: "component longer than 256 characters",
136+
},
137+
{
138+
ConnectionId: 1,
139+
Id: "test-long-component-block-500",
140+
IssueKey: issueKey,
141+
Component: longComponent500,
142+
Msg: "component longer than 500 characters",
143+
},
144+
}
145+
for _, block := range codeBlocks {
146+
require.NoError(t, dataflowTester.Db.Create(block).Error)
147+
}
148+
149+
dataflowTester.Subtask(tasks.ConvertIssueCodeBlocksMeta, &tasks.SonarqubeTaskData{
150+
Options: &tasks.SonarqubeOptions{
151+
ConnectionId: 1,
152+
ProjectKey: projectKey,
153+
},
154+
TaskStartTime: time.Now(),
155+
})
156+
157+
var domainBlocks []codequality.CqIssueCodeBlock
158+
require.NoError(t, dataflowTester.Db.Find(&domainBlocks).Error)
159+
require.Len(t, domainBlocks, 2)
160+
require.ElementsMatch(t,
161+
[]string{longComponent256, longComponent500},
162+
[]string{domainBlocks[0].Component, domainBlocks[1].Component},
163+
)
164+
165+
var toolBlocks []models.SonarqubeIssueCodeBlock
166+
require.NoError(t, dataflowTester.Db.Where(
167+
"connection_id = ? AND issue_key = ?", 1, issueKey,
168+
).Find(&toolBlocks).Error)
169+
require.Len(t, toolBlocks, 2)
170+
require.ElementsMatch(t,
171+
[]string{longComponent256, longComponent500},
172+
[]string{toolBlocks[0].Component, toolBlocks[1].Component},
173+
)
174+
}
175+
176+
func runMigration(t *testing.T, scripts []plugin.MigrationScript, name string, basicRes *implcontext.DefaultBasicRes) {
177+
t.Helper()
178+
for _, script := range scripts {
179+
if script.Name() == name {
180+
require.NoError(t, script.Up(basicRes))
181+
return
182+
}
183+
}
184+
require.Fail(t, "migration is not registered", name)
185+
}
186+
187+
func assertTextColumnWithoutIndex(t *testing.T, dataflowTester *e2ehelper.DataFlowTester, table string) {
188+
t.Helper()
189+
columnTypes, err := dataflowTester.Db.Migrator().ColumnTypes(table)
190+
require.NoError(t, err)
191+
for _, columnType := range columnTypes {
192+
if columnType.Name() == "component" {
193+
require.Contains(t, strings.ToLower(columnType.DatabaseTypeName()), "text")
194+
require.False(t, dataflowTester.Db.Migrator().HasIndex(table, "idx_"+table+"_component"))
195+
return
196+
}
197+
}
198+
require.Fail(t, "component column not found", table)
199+
}

‎backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/core/plugin"
24+
)
25+
26+
var _ plugin.MigrationScript = (*changeIssueCodeBlockComponentType)(nil)
27+
28+
type changeIssueCodeBlockComponentType struct{}
29+
30+
func (script *changeIssueCodeBlockComponentType) Up(basicRes context.BasicRes) errors.Error {
31+
db := basicRes.GetDal()
32+
if err := db.DropIndexes("_tool_sonarqube_issue_code_blocks", "idx__tool_sonarqube_issue_code_blocks_component"); err != nil {
33+
return err
34+
}
35+
return db.ModifyColumnType("_tool_sonarqube_issue_code_blocks", "component", "text")
36+
}
37+
38+
func (*changeIssueCodeBlockComponentType) Version() uint64 {
39+
return 20260701000000
40+
}
41+
42+
func (*changeIssueCodeBlockComponentType) Name() string {
43+
return "change _tool_sonarqube_issue_code_blocks.component type to text"
44+
}

‎backend/plugins/sonarqube/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ func All() []plugin.MigrationScript {
3939
new(addOrgToConn),
4040
new(addIssueImpacts),
4141
new(extendSonarqubeFieldSize),
42+
new(changeIssueCodeBlockComponentType),
4243
new(addProjectMetricsHistory),
4344
}
4445
}

‎backend/plugins/sonarqube/models/sonarqube_issue_code_block.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ type SonarqubeIssueCodeBlock struct {
2323
ConnectionId uint64 `gorm:"primaryKey"`
2424
Id string `gorm:"primaryKey"`
2525
IssueKey string `gorm:"index"`
26-
Component string `gorm:"index;type:varchar(500)"`
26+
Component string `gorm:"type:text"`
2727
StartLine int
2828
EndLine int
2929
StartOffset int

‎backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"reflect"
22+
"testing"
23+
24+
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
25+
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
26+
"github.com/stretchr/testify/require"
27+
)
28+
29+
func TestIssueCodeBlockComponentIsText(t *testing.T) {
30+
testCases := []struct {
31+
name string
32+
model interface{}
33+
}{
34+
{name: "tool layer", model: models.SonarqubeIssueCodeBlock{}},
35+
{name: "domain layer", model: codequality.CqIssueCodeBlock{}},
36+
}
37+
38+
for _, testCase := range testCases {
39+
t.Run(testCase.name, func(t *testing.T) {
40+
field, ok := reflect.TypeOf(testCase.model).FieldByName("Component")
41+
require.True(t, ok)
42+
require.Equal(t, "type:text", field.Tag.Get("gorm"))
43+
})
44+
}
45+
}

0 commit comments

Comments
 (0)