Skip to content

Commit 15e34bb

Browse files
jbsmith7741jbsmith7741
authored
fix(jira): collect issues via search API to avoid kanban sub-filter exclusion (apache#8931)
* feat(jira): collect issues via search API to avoid kanban sub-filter exclusion * Switch from board Agile API to search API with saved filter JQL so resolved issues (e.g. released fixVersions) are no longer silently excluded by kanban sub-filters * Add sub_query field to boards model and migration to track kanban sub-filter changes across syncs * Support JIRA Cloud v3 search/jql endpoint alongside v2 for Server * fix(jira,config-ui): correct SubQuery parsing and connection form crashes * Fix Jira board SubQuery struct to match nested API response shape * Prevent click event from being stored as plugin state in connection list * Default initialValues to empty object when plugin config omits it * feat(jira): Add BoardConfiguration unmarshal tests * Verify SubQuery parsing for kanban boards with, without, and empty sub-filter objects * Include a test using a full Jira Cloud response payload to validate all fields
1 parent 6f2ee9d commit 15e34bb

10 files changed

Lines changed: 361 additions & 79 deletions

File tree

‎backend/plugins/jira/models/board.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ type JiraBoard struct {
3434
Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"`
3535
Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"`
3636
Jql string `json:"jql" mapstructure:"jql"`
37+
SubQuery string `json:"subQuery" mapstructure:"subQuery"`
3738
}
3839

3940
func (b JiraBoard) ScopeId() string {

‎backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
24+
)
25+
26+
type JiraBoard20260611 struct {
27+
SubQuery string
28+
}
29+
30+
func (JiraBoard20260611) TableName() string {
31+
return "_tool_jira_boards"
32+
}
33+
34+
type addSubQueryToBoards struct{}
35+
36+
func (script *addSubQueryToBoards) Up(basicRes context.BasicRes) errors.Error {
37+
return migrationhelper.AutoMigrateTables(basicRes, &JiraBoard20260611{})
38+
}
39+
40+
func (*addSubQueryToBoards) Version() uint64 {
41+
return 20260611140000
42+
}
43+
44+
func (*addSubQueryToBoards) Name() string {
45+
return "add sub_query to _tool_jira_boards"
46+
}

‎backend/plugins/jira/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,5 +55,6 @@ func All() []plugin.MigrationScript {
5555
new(flushJiraIssues),
5656
new(updateScopeConfig),
5757
new(addFixVersions20250619),
58+
new(addSubQueryToBoards),
5859
}
5960
}

‎backend/plugins/jira/tasks/board_filter_begin_collector.go‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,18 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error {
4141
logger := taskCtx.GetLogger()
4242
db := taskCtx.GetDal()
4343
logger.Info("collect board in collectBoardFilterBegin: %d", data.Options.BoardId)
44-
// get board filter id
45-
filterId, err := getBoardFilterId(data)
44+
45+
boardConfig, err := getBoardConfiguration(data)
4646
if err != nil {
47-
return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
47+
return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
4848
}
49+
filterId := boardConfig.Filter.ID
4950
logger.Info("collect board filter:%s", filterId)
5051

51-
// get board filter jql
52+
if boardConfig.SubQuery.Query != "" {
53+
logger.Warn(nil, "board %d has kanban sub-filter: %s — using saved filter JQL for collection to avoid silent issue exclusion", data.Options.BoardId, boardConfig.SubQuery.Query)
54+
}
55+
5256
filterInfo, err := getBoardFilterJql(data, filterId)
5357
if err != nil {
5458
return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
@@ -62,17 +66,21 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error {
6266
return errors.Default.Wrap(err, fmt.Sprintf("error finding record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
6367
}
6468

69+
// Store filter ID and sub-query on task data for downstream subtasks
70+
data.FilterId = filterId
71+
record.SubQuery = boardConfig.SubQuery.Query
72+
6573
// full sync
6674
syncPolicy := taskCtx.TaskContext().SyncPolicy()
6775
if syncPolicy != nil && syncPolicy.FullSync {
6876
if record.Jql != jql {
6977
record.Jql = jql
70-
err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
71-
if err != nil {
72-
return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
73-
}
74-
logger.Info("full sync mode, update jql to %s", record.Jql)
7578
}
79+
err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
80+
if err != nil {
81+
return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
82+
}
83+
logger.Info("full sync mode, update jql to %s", record.Jql)
7684
return nil
7785
}
7886

@@ -92,7 +100,6 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error {
92100
flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
93101
if flag {
94102
logger.Info("connection_id:%d board_id:%d filter jql has changed, And the previous jql is %s, now jql is %s, run it in fullSync mode", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)
95-
// set full sync
96103
taskCtx.TaskContext().SetSyncPolicy(&coreModels.SyncPolicy{TriggerSyncPolicy: coreModels.TriggerSyncPolicy{FullSync: true}})
97104
record.Jql = jql
98105
err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
@@ -102,24 +109,28 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error {
102109
} else {
103110
return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql))
104111
}
112+
} else {
113+
// JQL unchanged but sub-query may have changed — persist it
114+
err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
115+
if err != nil {
116+
return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
117+
}
105118
}
106-
// no change
107119
return nil
108120
}
109121

110-
func getBoardFilterId(data *JiraTaskData) (string, error) {
122+
func getBoardConfiguration(data *JiraTaskData) (*BoardConfiguration, error) {
111123
url := fmt.Sprintf("agile/1.0/board/%d/configuration", data.Options.BoardId)
112124
boardConfiguration, err := data.ApiClient.Get(url, nil, nil)
113125
if err != nil {
114-
return "", err
126+
return nil, err
115127
}
116128
bc := &BoardConfiguration{}
117129
err = helper.UnmarshalResponse(boardConfiguration, bc)
118130
if err != nil {
119-
return "", err
131+
return nil, err
120132
}
121-
filterId := bc.Filter.ID
122-
return filterId, nil
133+
return bc, nil
123134
}
124135

125136
func getBoardFilterJql(data *JiraTaskData, filterId string) (*FilterInfo, error) {
@@ -141,6 +152,9 @@ type BoardConfiguration struct {
141152
Name string `json:"name"`
142153
Type string `json:"type"`
143154
Self string `json:"self"`
155+
SubQuery struct {
156+
Query string `json:"query"`
157+
} `json:"subQuery"`
144158
Location struct {
145159
Type string `json:"type"`
146160
Key string `json:"key"`

‎backend/plugins/jira/tasks/board_filter_begin_collector_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"encoding/json"
22+
"testing"
23+
)
24+
25+
func Test_BoardConfiguration_UnmarshalSubQuery(t *testing.T) {
26+
tests := []struct {
27+
name string
28+
raw string
29+
wantSubQuery string
30+
wantFilterID string
31+
wantID int
32+
wantName string
33+
wantType string
34+
wantColumnCount int
35+
wantRankFieldID int
36+
}{
37+
{
38+
name: "kanban board with sub-filter object",
39+
raw: `{"id":1201,"name":"Squad 5","type":"kanban",` +
40+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/1201/configuration",` +
41+
`"filter":{"id":"17696","self":"https://example.atlassian.net/rest/api/2/filter/17696"},` +
42+
`"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},` +
43+
`"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://example.atlassian.net/rest/api/2/status/1"}]},` +
44+
`{"name":"Done","statuses":[{"id":"10037","self":"https://example.atlassian.net/rest/api/2/status/10037"}]}],` +
45+
`"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
46+
wantSubQuery: "fixVersion in unreleasedVersions() OR fixVersion is EMPTY",
47+
wantFilterID: "17696",
48+
wantID: 1201,
49+
wantName: "Squad 5",
50+
wantType: "kanban",
51+
wantColumnCount: 2,
52+
wantRankFieldID: 10019,
53+
},
54+
{
55+
name: "board without subQuery field",
56+
raw: `{"id":500,"name":"No SubFilter Board","type":"scrum",` +
57+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/500/configuration",` +
58+
`"filter":{"id":"99999","self":"https://example.atlassian.net/rest/api/2/filter/99999"},` +
59+
`"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
60+
wantSubQuery: "",
61+
wantFilterID: "99999",
62+
wantID: 500,
63+
wantName: "No SubFilter Board",
64+
wantType: "scrum",
65+
wantColumnCount: 0,
66+
wantRankFieldID: 10019,
67+
},
68+
{
69+
name: "board with empty subQuery object",
70+
raw: `{"id":600,"name":"Empty SubQuery Board","type":"kanban",` +
71+
`"self":"https://example.atlassian.net/rest/agile/1.0/board/600/configuration",` +
72+
`"filter":{"id":"11111","self":"https://example.atlassian.net/rest/api/2/filter/11111"},` +
73+
`"subQuery":{},` +
74+
`"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`,
75+
wantSubQuery: "",
76+
wantFilterID: "11111",
77+
wantID: 600,
78+
wantName: "Empty SubQuery Board",
79+
wantType: "kanban",
80+
wantColumnCount: 0,
81+
wantRankFieldID: 10019,
82+
},
83+
}
84+
85+
for _, tt := range tests {
86+
t.Run(tt.name, func(t *testing.T) {
87+
var bc BoardConfiguration
88+
if err := json.Unmarshal([]byte(tt.raw), &bc); err != nil {
89+
t.Fatalf("failed to unmarshal BoardConfiguration: %v", err)
90+
}
91+
if bc.SubQuery.Query != tt.wantSubQuery {
92+
t.Errorf("SubQuery.Query = %q, want %q", bc.SubQuery.Query, tt.wantSubQuery)
93+
}
94+
if bc.Filter.ID != tt.wantFilterID {
95+
t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, tt.wantFilterID)
96+
}
97+
if bc.ID != tt.wantID {
98+
t.Errorf("ID = %d, want %d", bc.ID, tt.wantID)
99+
}
100+
if bc.Name != tt.wantName {
101+
t.Errorf("Name = %q, want %q", bc.Name, tt.wantName)
102+
}
103+
if bc.Type != tt.wantType {
104+
t.Errorf("Type = %q, want %q", bc.Type, tt.wantType)
105+
}
106+
if len(bc.ColumnConfig.Columns) != tt.wantColumnCount {
107+
t.Errorf("ColumnConfig.Columns length = %d, want %d", len(bc.ColumnConfig.Columns), tt.wantColumnCount)
108+
}
109+
if bc.Ranking.RankCustomFieldID != tt.wantRankFieldID {
110+
t.Errorf("Ranking.RankCustomFieldID = %d, want %d", bc.Ranking.RankCustomFieldID, tt.wantRankFieldID)
111+
}
112+
})
113+
}
114+
}
115+
116+
func Test_BoardConfiguration_FullJiraCloudResponse(t *testing.T) {
117+
// Exact response payload from Jira Cloud for Board 1201 (Squad 5)
118+
raw := `{"id":1201,"name":"Squad 5","type":"kanban","self":"https://rakutenadvertising.atlassian.net/rest/agile/1.0/board/1201/configuration","location":{"type":"user","id":"62d8159bb2e6b1992b5be875","self":"https://rakutenadvertising.atlassian.net/rest/api/2/user?accountId=62d8159bb2e6b1992b5be875"},"filter":{"id":"17696","self":"https://rakutenadvertising.atlassian.net/rest/api/2/filter/17696"},"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/1"},{"id":"4","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/4"},{"id":"10016","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10016"},{"id":"10003","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10003"}]},{"name":"To Do","statuses":[{"id":"10054","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10054"}]},{"name":"Blocked","statuses":[{"id":"10019","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10019"}]},{"name":"In Development","statuses":[{"id":"10017","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10017"},{"id":"10177","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10177"},{"id":"10038","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10038"}]},{"name":"Code Review","statuses":[{"id":"10024","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10024"}]},{"name":"Ready for QA","statuses":[{"id":"10029","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10029"},{"id":"10033","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10033"}]},{"name":"In QA","statuses":[{"id":"10018","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10018"},{"id":"10158","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10158"}]},{"name":"Done","statuses":[{"id":"10037","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10037"}]}],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`
119+
120+
var bc BoardConfiguration
121+
if err := json.Unmarshal([]byte(raw), &bc); err != nil {
122+
t.Fatalf("failed to unmarshal real Jira Cloud response: %v", err)
123+
}
124+
125+
if bc.SubQuery.Query != "fixVersion in unreleasedVersions() OR fixVersion is EMPTY" {
126+
t.Errorf("SubQuery.Query = %q, want the fixVersion sub-filter", bc.SubQuery.Query)
127+
}
128+
if bc.Filter.ID != "17696" {
129+
t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, "17696")
130+
}
131+
if len(bc.ColumnConfig.Columns) != 8 {
132+
t.Errorf("ColumnConfig.Columns length = %d, want 8", len(bc.ColumnConfig.Columns))
133+
}
134+
if bc.Location.ID != "62d8159bb2e6b1992b5be875" {
135+
t.Errorf("Location.ID = %q, want %q", bc.Location.ID, "62d8159bb2e6b1992b5be875")
136+
}
137+
}

‎backend/plugins/jira/tasks/board_filter_end_collector.go‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,35 +40,42 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error {
4040
db := taskCtx.GetDal()
4141
logger.Info("collect board in collectBoardFilterEnd: %d", data.Options.BoardId)
4242

43-
// get board filter id
44-
filterId, err := getBoardFilterId(data)
43+
boardConfig, err := getBoardConfiguration(data)
4544
if err != nil {
46-
return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
45+
return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
4746
}
47+
filterId := boardConfig.Filter.ID
4848
logger.Info("collect board filter:%s", filterId)
4949

50-
// get board filter jql
5150
filterInfo, err := getBoardFilterJql(data, filterId)
5251
if err != nil {
5352
return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
5453
}
5554
jql := filterInfo.Jql
5655
logger.Info("collect board filter jql:%s", jql)
5756

58-
// should not change
5957
var record models.JiraBoard
6058
err = db.First(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId))
6159
if err != nil {
6260
return errors.Default.Wrap(err, fmt.Sprintf("error finding record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId))
6361
}
6462
logger.Info("get board filter jql:%s", record.Jql)
6563

64+
cfg := taskCtx.GetConfigReader()
65+
autoRefresh := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
66+
6667
if record.Jql != jql {
67-
cfg := taskCtx.GetConfigReader()
68-
flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH")
69-
if !flag {
68+
if !autoRefresh {
7069
return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql))
7170
}
71+
logger.Warn(nil, "connection_id:%d board_id:%d filter jql changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)
72+
}
73+
74+
if record.SubQuery != boardConfig.SubQuery.Query {
75+
logger.Warn(nil, "connection_id:%d board_id:%d board sub-filter changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query)
76+
if !autoRefresh {
77+
return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d board sub-filter has changed during collection, please use fullSync mode. Previous sub-filter: %s, now: %s", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query))
78+
}
7279
}
7380

7481
return nil

0 commit comments

Comments
 (0)