Skip to content

Commit 1cbd835

Browse files
jbsmith7741jbsmith7741
authored
feat(sonarqube): collect historical project metrics and Grafana trends (apache#8983)
* feat(sonarqube): add project-level metrics history collection Add collector, extractor, and convertor subtasks that pull historical metric snapshots from the SonarQube measures/search_history and project_analyses/search APIs into a new cq_project_metrics_history domain table, enabling trend analysis for coverage, bugs, code smells, complexity, vulnerabilities, and other quality indicators over time. Signed-off-by: Joshua Smith <jbsmith7741@gmail.com> * fix(sonarqube): Require user token at test * Reject Global and Project analysis tokens on SonarQube Server before authentication validate runs. * Skip prefix validation for SonarCloud endpoints where token prefixes differ. * feat(grafana): add historical trends to SonarQube dashboards * Add collapsed "Historical Trends" row with coverage, bugs, vulnerabilities, code smells, and duplication time-series panels to both Server and Cloud dashboards * Panels query cq_project_metrics_history and honor the Grafana time picker via $__timeFilter Signed-off-by: Joshua Smith <jbsmith7741@gmail.com> * style(sonarqube): Fix gofmt formatting in code quality files * Align struct fields and ApiCollectorArgs literals to satisfy golangci-lint gofmt checks. * Updates cq_project_metrics_history and SonarQube collector tasks flagged in CI. --------- Signed-off-by: Joshua Smith <jbsmith7741@gmail.com>
1 parent 3a4333e commit 1cbd835

22 files changed

Lines changed: 1999 additions & 9 deletions

‎backend/core/models/domainlayer/codequality/cq_project_metrics_history.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package codequality
19+
20+
import (
21+
"time"
22+
23+
"github.com/apache/incubator-devlake/core/models/domainlayer"
24+
)
25+
26+
type CqProjectMetricsHistory struct {
27+
domainlayer.DomainEntity
28+
ProjectKey string `gorm:"index;type:varchar(500)"`
29+
AnalysisDate time.Time `gorm:"index"`
30+
Coverage *float64
31+
Ncloc *int
32+
Bugs *int
33+
ReliabilityRating string `gorm:"type:varchar(5)"`
34+
CodeSmells *int
35+
SqaleRating string `gorm:"type:varchar(5)"`
36+
Complexity *int
37+
CognitiveComplexity *int
38+
Vulnerabilities *int
39+
SecurityRating string `gorm:"type:varchar(5)"`
40+
SecurityHotspots *int
41+
DuplicatedLinesDensity *float64
42+
}
43+
44+
func (CqProjectMetricsHistory) TableName() string {
45+
return "cq_project_metrics_history"
46+
}

‎backend/core/models/domainlayer/domaininfo/domaininfo.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ func GetDomainTablesInfo() []dal.Tabler {
5656
&codequality.CqIssue{},
5757
&codequality.CqIssueImpact{},
5858
&codequality.CqProject{},
59+
&codequality.CqProjectMetricsHistory{},
5960
// crossdomain
6061
&crossdomain.Account{},
6162
&crossdomain.BoardRepo{},

‎backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
24+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
25+
)
26+
27+
type addCqProjectMetricsHistory struct{}
28+
29+
func (u *addCqProjectMetricsHistory) Up(basicRes context.BasicRes) errors.Error {
30+
return migrationhelper.AutoMigrateTables(
31+
basicRes,
32+
&archived.CqProjectMetricsHistory{},
33+
)
34+
}
35+
36+
func (*addCqProjectMetricsHistory) Version() uint64 {
37+
return 20260707153201
38+
}
39+
40+
func (*addCqProjectMetricsHistory) Name() string {
41+
return "add cq_project_metrics_history domain table"
42+
}

‎backend/core/models/migrationscripts/archived/cq_project_metrics_history.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package archived
19+
20+
import "time"
21+
22+
type CqProjectMetricsHistory struct {
23+
DomainEntity
24+
ProjectKey string `gorm:"index;type:varchar(500)"`
25+
AnalysisDate time.Time `gorm:"index"`
26+
Coverage *float64
27+
Ncloc *int
28+
Bugs *int
29+
ReliabilityRating string `gorm:"type:varchar(5)"`
30+
CodeSmells *int
31+
SqaleRating string `gorm:"type:varchar(5)"`
32+
Complexity *int
33+
CognitiveComplexity *int
34+
Vulnerabilities *int
35+
SecurityRating string `gorm:"type:varchar(5)"`
36+
SecurityHotspots *int
37+
DuplicatedLinesDensity *float64
38+
}
39+
40+
func (CqProjectMetricsHistory) TableName() string {
41+
return "cq_project_metrics_history"
42+
}

‎backend/core/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,5 +145,6 @@ func All() []plugin.MigrationScript {
145145
new(modifyCicdDeploymentsToText),
146146
new(increaseCqIssuesProjectKeyLength),
147147
new(addAuthSessions),
148+
new(addCqProjectMetricsHistory),
148149
}
149150
}

‎backend/plugins/sonarqube/api/connection_api.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ func testConnection(ctx context.Context, connection models.SonarqubeConn) (*plug
4444
return nil, errors.Default.Wrap(err, "error validating target")
4545
}
4646
}
47+
if err := connection.ValidateUserTokenPrefix(); err != nil {
48+
return nil, err
49+
}
4750
apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection)
4851
if err != nil {
4952
return nil, err

‎backend/plugins/sonarqube/impl/impl.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {
8686
&models.SonarqubeFileMetrics{},
8787
&models.SonarqubeAccount{},
8888
&models.SonarqubeScopeConfig{},
89+
&models.SonarqubeProjectMetricsHistory{},
90+
&models.SonarqubeProjectAnalysis{},
8991
}
9092
}
9193

@@ -108,6 +110,11 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
108110
tasks.ConvertHotspotsMeta,
109111
tasks.ConvertFileMetricsMeta,
110112
tasks.ConvertAccountsMeta,
113+
tasks.CollectProjectMetricsHistoryMeta,
114+
tasks.ExtractProjectMetricsHistoryMeta,
115+
tasks.CollectProjectAnalysesMeta,
116+
tasks.ExtractProjectAnalysesMeta,
117+
tasks.ConvertProjectMetricsHistoryMeta,
111118
}
112119
}
113120

‎backend/plugins/sonarqube/models/connection.go‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"encoding/base64"
2222
"fmt"
2323
"net/http"
24+
"strings"
2425

2526
"github.com/apache/incubator-devlake/core/utils"
2627

@@ -29,6 +30,12 @@ import (
2930
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
3031
)
3132

33+
const (
34+
userTokenPrefix = "squ_"
35+
globalAnalysisTokenPrefix = "sqa_"
36+
projectAnalysisTokenPrefix = "sqp_"
37+
)
38+
3239
type SonarqubeAccessToken helper.AccessToken
3340

3441
// SetupAuthentication sets up the HTTP Request Authentication
@@ -92,7 +99,40 @@ func (connection *SonarqubeConnection) MergeFromRequest(target *SonarqubeConnect
9299
return nil
93100
}
94101

95-
func (connection *SonarqubeConnection) IsCloud() bool {
102+
// ValidateUserTokenPrefix ensures the token is a SonarQube User token on Server
103+
// instances. Global (sqa_) and Project (sqp_) analysis tokens authenticate but
104+
// cannot call read APIs such as measures/component_tree.
105+
func (connection SonarqubeConn) ValidateUserTokenPrefix() errors.Error {
106+
if connection.IsCloud() {
107+
return nil
108+
}
109+
token := strings.TrimSpace(connection.Token)
110+
if token == "" {
111+
return errors.BadInput.New("token is required")
112+
}
113+
switch {
114+
case strings.HasPrefix(token, globalAnalysisTokenPrefix):
115+
return errors.BadInput.New(
116+
"DevLake requires a User token (squ_ prefix). " +
117+
"Global Analysis tokens (sqa_) can push scan results but cannot read project metrics via the Web API. " +
118+
"Create a User token under My Account > Security in SonarQube.",
119+
)
120+
case strings.HasPrefix(token, projectAnalysisTokenPrefix):
121+
return errors.BadInput.New(
122+
"DevLake requires a User token (squ_ prefix). " +
123+
"Project Analysis tokens (sqp_) can push scan results but cannot read project metrics via the Web API. " +
124+
"Create a User token under My Account > Security in SonarQube.",
125+
)
126+
case !strings.HasPrefix(token, userTokenPrefix):
127+
return errors.BadInput.New(
128+
"DevLake requires a User token (squ_ prefix) for SonarQube Server. " +
129+
"Create one under My Account > Security in SonarQube.",
130+
)
131+
}
132+
return nil
133+
}
134+
135+
func (connection SonarqubeConn) IsCloud() bool {
96136
return connection.Endpoint == "https://sonarcloud.io/api/"
97137
}
98138

‎backend/plugins/sonarqube/models/connection_test.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package models
19+
20+
import (
21+
"testing"
22+
23+
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
24+
)
25+
26+
func TestValidateUserTokenPrefix(t *testing.T) {
27+
t.Parallel()
28+
29+
serverEndpoint := "https://rad-sonar.example.com/api/"
30+
cloudEndpoint := "https://sonarcloud.io/api/"
31+
32+
tests := []struct {
33+
name string
34+
conn SonarqubeConn
35+
wantErr bool
36+
}{
37+
{
38+
name: "user token on server",
39+
conn: SonarqubeConn{
40+
RestConnection: helper.RestConnection{Endpoint: serverEndpoint},
41+
SonarqubeAccessToken: SonarqubeAccessToken{Token: "squ_abc123"},
42+
},
43+
},
44+
{
45+
name: "global analysis token on server",
46+
conn: SonarqubeConn{
47+
RestConnection: helper.RestConnection{Endpoint: serverEndpoint},
48+
SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"},
49+
},
50+
wantErr: true,
51+
},
52+
{
53+
name: "project analysis token on server",
54+
conn: SonarqubeConn{
55+
RestConnection: helper.RestConnection{Endpoint: serverEndpoint},
56+
SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqp_abc123"},
57+
},
58+
wantErr: true,
59+
},
60+
{
61+
name: "unknown prefix on server",
62+
conn: SonarqubeConn{
63+
RestConnection: helper.RestConnection{Endpoint: serverEndpoint},
64+
SonarqubeAccessToken: SonarqubeAccessToken{Token: "legacy-token"},
65+
},
66+
wantErr: true,
67+
},
68+
{
69+
name: "sonarcloud skips prefix check",
70+
conn: SonarqubeConn{
71+
RestConnection: helper.RestConnection{Endpoint: cloudEndpoint},
72+
SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"},
73+
},
74+
},
75+
}
76+
77+
for _, tt := range tests {
78+
t.Run(tt.name, func(t *testing.T) {
79+
t.Parallel()
80+
err := tt.conn.ValidateUserTokenPrefix()
81+
if tt.wantErr {
82+
if err == nil {
83+
t.Fatal("expected error, got nil")
84+
}
85+
return
86+
}
87+
if err != nil {
88+
t.Fatalf("unexpected error: %v", err)
89+
}
90+
})
91+
}
92+
}

0 commit comments

Comments
 (0)