Skip to content

Commit 5b6f340

Browse files
mathur07mathur07
committed
feat: add error metrics for PipelineRun and TaskRun signing failures
This commit adds new error metrics to track signing failures for both PipelineRun and TaskRun artifacts. - Error recording in the signing logic with error_type tags. - Updated metrics and tests in both pipelinerunmetrics and taskrunmetrics.
1 parent 7824cdc commit 5b6f340

6 files changed

Lines changed: 158 additions & 32 deletions

File tree

‎pkg/chains/constants.go‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,28 @@ limitations under the License.
1414
package chains
1515

1616
const (
17-
SignedMessagesCount = "sgcount"
18-
SignsStoredCount = "stcount"
19-
PayloadUploadeCount = "plcount"
20-
MarkedAsSignedCount = "mrcount"
21-
PipelineRunSignedName = "pipelinerun_sign_created_total"
22-
PipelineRunSignedDesc = "Total number of signed messages for pipelineruns"
23-
PipelineRunUploadedName = "pipelinerun_payload_uploaded_total"
24-
PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns"
25-
PipelineRunStoredName = "pipelinerun_payload_stored_total"
26-
PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns"
27-
PipelineRunMarkedName = "pipelinerun_marked_signed_total"
28-
PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns"
29-
TaskRunSignedName = "taskrun_sign_created_total"
30-
TaskRunSignedDesc = "Total number of signed messages for taskruns"
31-
TaskRunUploadedName = "taskrun_payload_uploaded_total"
32-
TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns"
33-
TaskRunStoredName = "taskrun_payload_stored_total"
34-
TaskRunStoredDesc = "Total number of stored payloads for taskruns"
35-
TaskRunMarkedName = "taskrun_marked_signed_total"
36-
TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns"
17+
SignedMessagesCount = "sgcount"
18+
SignsStoredCount = "stcount"
19+
PayloadUploadeCount = "plcount"
20+
MarkedAsSignedCount = "mrcount"
21+
PipelineRunSignedName = "pipelinerun_sign_created_total"
22+
PipelineRunSignedDesc = "Total number of signed messages for pipelineruns"
23+
PipelineRunUploadedName = "pipelinerun_payload_uploaded_total"
24+
PipelineRunUploadedDesc = "Total number of uploaded payloads for pipelineruns"
25+
PipelineRunStoredName = "pipelinerun_payload_stored_total"
26+
PipelineRunStoredDesc = "Total number of stored payloads for pipelineruns"
27+
PipelineRunMarkedName = "pipelinerun_marked_signed_total"
28+
PipelineRunMarkedDesc = "Total number of objects marked as signed for pipelineruns"
29+
PipelineRunErrorCountName = "pipelinerun_signing_failures_total"
30+
PipelineRunErrorCountDesc = "Total number of PipelineRun signing failures"
31+
TaskRunSignedName = "taskrun_sign_created_total"
32+
TaskRunSignedDesc = "Total number of signed messages for taskruns"
33+
TaskRunUploadedName = "taskrun_payload_uploaded_total"
34+
TaskRunUploadedDesc = "Total number of uploaded payloads for taskruns"
35+
TaskRunStoredName = "taskrun_payload_stored_total"
36+
TaskRunStoredDesc = "Total number of stored payloads for taskruns"
37+
TaskRunMarkedName = "taskrun_marked_signed_total"
38+
TaskRunMarkedDesc = "Total number of objects marked as signed for taskruns"
39+
TaskRunErrorCountName = "taskrun_signing_failures_total"
40+
TaskRunErrorCountDesc = "Total number of TaskRun signing failures"
3741
)

‎pkg/chains/signing.go‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ type Signer interface {
4242

4343
type MetricsRecorder interface {
4444
RecordCountMetrics(ctx context.Context, MetricType string)
45+
RecordErrorMetric(ctx context.Context, errType string)
4546
}
4647

4748
type ObjectSigner struct {
@@ -114,8 +115,8 @@ func getSignableTypes(ctx context.Context, obj objects.TektonObject) ([]artifact
114115
return types, nil
115116
}
116117

117-
// Signs TaskRun and PipelineRun objects, as well as generates attesations for each
118-
// Follows process of extract payload, sign payload, store payload and signature
118+
// Signs TaskRun and PipelineRun objects, as well as generates attestations for each.
119+
// Follows process of extract payload, sign payload, store payload and signature.
119120
func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject) error {
120121
cfg := *config.FromContext(ctx)
121122
logger := logging.FromContext(ctx)
@@ -150,6 +151,9 @@ func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject)
150151
payload, err := payloader.CreatePayload(ctx, obj)
151152
if err != nil {
152153
logger.Error(err)
154+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
155+
o.Recorder.RecordErrorMetric(ctx, "payload_creation")
156+
}
153157
continue
154158
}
155159
logger.Infof("Created payload of type %s for %s %s/%s", string(payloadFormat), tektonObj.GetGVK(), tektonObj.GetNamespace(), tektonObj.GetName())
@@ -174,13 +178,19 @@ func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject)
174178
logger.Infof("Signing object with %s", signerType)
175179
rawPayload, err := getRawPayload(payload)
176180
if err != nil {
177-
logger.Warnf("Unable to marshal payload: %v", signerType, obj)
181+
logger.Warnf("Unable to marshal payload for %s: %v", signerType, err)
182+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
183+
o.Recorder.RecordErrorMetric(ctx, "marshal_payload")
184+
}
178185
continue
179186
}
180187

181188
signature, err := signer.SignMessage(bytes.NewReader(rawPayload))
182189
if err != nil {
183190
logger.Error(err)
191+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
192+
o.Recorder.RecordErrorMetric(ctx, "signing")
193+
}
184194
continue
185195
}
186196
measureMetrics(ctx, SignedMessagesCount, o.Recorder)
@@ -191,6 +201,9 @@ func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject)
191201
if !ok {
192202
backendErr := fmt.Errorf("could not find backend '%s' in configured backends (%v) while trying sign: %s/%s", backend, maps.Keys(o.Backends), tektonObj.GetKindName(), tektonObj.GetName())
193203
logger.Error(backendErr)
204+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
205+
o.Recorder.RecordErrorMetric(ctx, "storage")
206+
}
194207
merr = multierror.Append(merr, backendErr)
195208
continue
196209
}
@@ -204,6 +217,9 @@ func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject)
204217
}
205218
if err := b.StorePayload(ctx, tektonObj, rawPayload, string(signature), storageOpts); err != nil {
206219
logger.Error(err)
220+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
221+
o.Recorder.RecordErrorMetric(ctx, "storage")
222+
}
207223
merr = multierror.Append(merr, err)
208224
} else {
209225
measureMetrics(ctx, SignsStoredCount, o.Recorder)
@@ -219,6 +235,9 @@ func (o *ObjectSigner) Sign(ctx context.Context, tektonObj objects.TektonObject)
219235
entry, err := rekorClient.UploadTlog(ctx, signer, signature, rawPayload, signer.Cert(), string(payloadFormat))
220236
if err != nil {
221237
logger.Warnf("error uploading entry to tlog: %v", err)
238+
if (signableType.Type() == "TaskRunArtifact" || signableType.Type() == "PipelineRunArtifact") && o.Recorder != nil {
239+
o.Recorder.RecordErrorMetric(ctx, "tlog")
240+
}
222241
merr = multierror.Append(merr, err)
223242
} else {
224243
logger.Infof("Uploaded entry to %s with index %d", cfg.Transparency.URL, *entry.LogIndex)
@@ -249,7 +268,6 @@ func measureMetrics(ctx context.Context, metrictype string, mtr MetricsRecorder)
249268
if mtr != nil {
250269
mtr.RecordCountMetrics(ctx, metrictype)
251270
}
252-
253271
}
254272

255273
func HandleRetry(ctx context.Context, obj objects.TektonObject, ps versioned.Interface, annotations map[string]string) error {

‎pkg/pipelinerunmetrics/metrics.go‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/tektoncd/chains/pkg/chains"
2424
"go.opencensus.io/stats"
2525
"go.opencensus.io/stats/view"
26+
"go.opencensus.io/tag"
2627
"knative.dev/pkg/logging"
2728
"knative.dev/pkg/metrics"
2829
)
@@ -51,9 +52,22 @@ var (
5152
stats.UnitDimensionless)
5253

5354
mrCountView *view.View
55+
56+
// New error metric for PipelineRun signing failures, defined as a Float64 measure.
57+
pipelineRunErrorCount = stats.Float64(
58+
chains.PipelineRunErrorCountName,
59+
chains.PipelineRunErrorCountDesc,
60+
stats.UnitDimensionless,
61+
)
62+
63+
// Define a view for the PipelineRun error metric with an error_type tag.
64+
pipelineErrorView *view.View
65+
66+
// Tag key for error type.
67+
errorTypeKey, _ = tag.NewKey("error_type")
5468
)
5569

56-
// Recorder holds keys for Tekton metrics
70+
// Recorder holds keys for PipelineRun metrics.
5771
type Recorder struct {
5872
initialized bool
5973
}
@@ -67,7 +81,7 @@ var (
6781
)
6882

6983
// NewRecorder creates a new metrics recorder instance
70-
// to log the PipelineRun related metrics
84+
// to log the PipelineRun related metrics.
7185
func NewRecorder(ctx context.Context) (*Recorder, error) {
7286
var errRegistering error
7387
logger := logging.FromContext(ctx)
@@ -110,11 +124,21 @@ func viewRegister() error {
110124
Measure: mrCount,
111125
Aggregation: view.Count(),
112126
}
127+
128+
pipelineErrorView = &view.View{
129+
Name: chains.PipelineRunErrorCountName,
130+
Description: pipelineRunErrorCount.Description(),
131+
Measure: pipelineRunErrorCount,
132+
TagKeys: []tag.Key{errorTypeKey},
133+
Aggregation: view.Count(),
134+
}
135+
113136
return view.Register(
114137
sgCountView,
115138
plCountView,
116139
stCountView,
117140
mrCountView,
141+
pipelineErrorView,
118142
)
119143
}
120144

@@ -136,9 +160,15 @@ func (r *Recorder) RecordCountMetrics(ctx context.Context, metricType string) {
136160
default:
137161
logger.Errorf("Ignoring the metrics recording as valid Metric type matching %v was not found", mt)
138162
}
139-
140163
}
141164

142165
func (r *Recorder) countMetrics(ctx context.Context, measure *stats.Float64Measure) {
143166
metrics.Record(ctx, measure.M(1))
144167
}
168+
169+
// RecordErrorMetric records a PipelineRun signing failure with a given error type tag.
170+
func (r *Recorder) RecordErrorMetric(ctx context.Context, errType string) {
171+
// Add the error_type tag to the context.
172+
ctx, _ = tag.New(ctx, tag.Upsert(errorTypeKey, errType))
173+
metrics.Record(ctx, pipelineRunErrorCount.M(1))
174+
}

‎pkg/pipelinerunmetrics/metrics_test.go‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,29 @@ func TestCountMetrics(t *testing.T) {
6060
metricstest.CheckCountData(t, chains.PipelineRunMarkedName, map[string]string{}, 1)
6161
}
6262

63+
func TestRecordErrorMetric(t *testing.T) {
64+
unregisterMetrics()
65+
ctx := context.Background()
66+
ctx = WithClient(ctx)
67+
68+
rec := Get(ctx)
69+
if rec == nil {
70+
t.Fatal("Recorder not initialized")
71+
}
72+
73+
// Record an error metric with a sample error type "signing"
74+
rec.RecordErrorMetric(ctx, "signing")
75+
metricstest.CheckCountData(t, chains.PipelineRunErrorCountName, map[string]string{"error_type": "signing"}, 1)
76+
}
77+
6378
func unregisterMetrics() {
64-
metricstest.Unregister(chains.PipelineRunSignedName, chains.PipelineRunUploadedName, chains.PipelineRunStoredName, chains.PipelineRunMarkedName)
79+
metricstest.Unregister(
80+
chains.PipelineRunSignedName,
81+
chains.PipelineRunUploadedName,
82+
chains.PipelineRunStoredName,
83+
chains.PipelineRunMarkedName,
84+
chains.PipelineRunErrorCountName,
85+
)
6586
// Allow the recorder singleton to be recreated.
6687
once = sync.Once{}
6788
r = nil

‎pkg/taskrunmetrics/metrics.go‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/tektoncd/chains/pkg/chains"
2424
"go.opencensus.io/stats"
2525
"go.opencensus.io/stats/view"
26+
"go.opencensus.io/tag"
2627
"knative.dev/pkg/logging"
2728
"knative.dev/pkg/metrics"
2829
)
@@ -51,9 +52,22 @@ var (
5152
stats.UnitDimensionless)
5253

5354
mrCountView *view.View
55+
56+
// New error metric for TaskRun signing failures as a Float64 measure.
57+
taskRunErrorCount = stats.Float64(
58+
chains.TaskRunErrorCountName,
59+
chains.TaskRunErrorCountDesc,
60+
stats.UnitDimensionless,
61+
)
62+
63+
// Define a view for the error metric with a tag for error type.
64+
errorCountView *view.View
65+
66+
// Tag key for error type.
67+
errorTypeKey, _ = tag.NewKey("error_type")
5468
)
5569

56-
// Recorder is used to actually record TaskRun metrics
70+
// Recorder is used to actually record TaskRun metrics.
5771
type Recorder struct {
5872
initialized bool
5973
}
@@ -67,7 +81,7 @@ var (
6781
)
6882

6983
// NewRecorder creates a new metrics recorder instance
70-
// to log the TaskRun related metrics
84+
// to log the TaskRun related metrics.
7185
func NewRecorder(ctx context.Context) (*Recorder, error) {
7286
var errRegistering error
7387
logger := logging.FromContext(ctx)
@@ -111,11 +125,21 @@ func viewRegister() error {
111125
Measure: mrCount,
112126
Aggregation: view.Count(),
113127
}
128+
129+
errorCountView = &view.View{
130+
Name: chains.TaskRunErrorCountName,
131+
Description: taskRunErrorCount.Description(),
132+
Measure: taskRunErrorCount,
133+
TagKeys: []tag.Key{errorTypeKey},
134+
Aggregation: view.Count(),
135+
}
136+
114137
return view.Register(
115138
sgCountView,
116139
plCountView,
117140
stCountView,
118141
mrCountView,
142+
errorCountView,
119143
)
120144
}
121145

@@ -137,9 +161,15 @@ func (r *Recorder) RecordCountMetrics(ctx context.Context, metricType string) {
137161
default:
138162
logger.Errorf("Ignoring the metrics recording as valid Metric type matching %v was not found", mt)
139163
}
140-
141164
}
142165

143166
func (r *Recorder) countMetrics(ctx context.Context, measure *stats.Float64Measure) {
144167
metrics.Record(ctx, measure.M(1))
145168
}
169+
170+
// RecordErrorMetric records a TaskRun signing failure with a given error type tag.
171+
func (r *Recorder) RecordErrorMetric(ctx context.Context, errType string) {
172+
// Add the error_type tag to the context.
173+
ctx, _ = tag.New(ctx, tag.Upsert(errorTypeKey, errType))
174+
metrics.Record(ctx, taskRunErrorCount.M(1))
175+
}

‎pkg/taskrunmetrics/metrics_test.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,31 @@ func TestCountMetrics(t *testing.T) {
6060
metricstest.CheckCountData(t, chains.TaskRunMarkedName, map[string]string{}, 1)
6161
}
6262

63+
func TestRecordErrorMetric(t *testing.T) {
64+
unregisterMetrics()
65+
ctx := context.Background()
66+
ctx = WithClient(ctx)
67+
68+
rec := Get(ctx)
69+
if rec == nil {
70+
t.Fatal("Recorder not initialized")
71+
}
72+
73+
// Record an error metric with a sample error type "signing"
74+
rec.RecordErrorMetric(ctx, "signing")
75+
76+
// Verify that the error metric is recorded with the tag error_type=signing.
77+
metricstest.CheckCountData(t, chains.TaskRunErrorCountName, map[string]string{"error_type": "signing"}, 1)
78+
}
79+
6380
func unregisterMetrics() {
64-
metricstest.Unregister(chains.TaskRunSignedName, chains.TaskRunUploadedName, chains.TaskRunStoredName, chains.TaskRunMarkedName)
81+
metricstest.Unregister(
82+
chains.TaskRunSignedName,
83+
chains.TaskRunUploadedName,
84+
chains.TaskRunStoredName,
85+
chains.TaskRunMarkedName,
86+
chains.TaskRunErrorCountName,
87+
)
6588
// Allow the recorder singleton to be recreated.
6689
once = sync.Once{}
6790
r = nil

0 commit comments

Comments
 (0)