[Metrics] Histogram for deadline remaining on Append (#737)

In sigstore/rekor-tiles#434 it is reported that there are deadline exceeded errors. This could be Tessera taking longer than expected, or it could be clients calling this with unreasonably short deadlines. This histogram metric should give some insight into the spectrum of deadlines that clients are expecting Tessera to work with.

Author: mhutchinson

append_lifecycle.go

@@ -51,15 +51,16 @@ const (
 )
 
 var (
-	appenderAddsTotal        metric.Int64Counter
-	appenderAddHistogram     metric.Int64Histogram
-	appenderHighestIndex     metric.Int64Gauge
-	appenderIntegratedSize   metric.Int64Gauge
-	appenderIntegrateLatency metric.Int64Histogram
-	appenderNextIndex        metric.Int64Gauge
-	appenderSignedSize       metric.Int64Gauge
-	appenderWitnessedSize    metric.Int64Gauge
-	appenderWitnessRequests  metric.Int64Counter
+	appenderAddsTotal         metric.Int64Counter
+	appenderAddHistogram      metric.Int64Histogram
+	appenderHighestIndex      metric.Int64Gauge
+	appenderIntegratedSize    metric.Int64Gauge
+	appenderIntegrateLatency  metric.Int64Histogram
+	appenderDeadlineRemaining metric.Int64Histogram
+	appenderNextIndex         metric.Int64Gauge
+	appenderSignedSize        metric.Int64Gauge
+	appenderWitnessedSize     metric.Int64Gauge
+	appenderWitnessRequests   metric.Int64Counter
 
 	followerEntriesProcessed metric.Int64Gauge
 	followerLag              metric.Int64Gauge
@@ -112,6 +113,15 @@ func init() {
 		klog.Exitf("Failed to create appenderIntegrateLatency metric: %v", err)
 	}
 
+	appenderDeadlineRemaining, err = meter.Int64Histogram(
+		"tessera.appender.deadline.remaining",
+		metric.WithDescription("Duration remaining before context cancellation when appender is invoked (only set for contexts with deadline)"),
+		metric.WithUnit("ms"),
+		metric.WithExplicitBucketBoundaries(histogramBuckets...))
+	if err != nil {
+		klog.Exitf("Failed to create appenderDeadlineRemaining metric: %v", err)
+	}
+
 	appenderNextIndex, err = meter.Int64Gauge(
 		"tessera.appender.next_index",
 		metric.WithDescription("The next available index to be assigned to entries"))
@@ -255,6 +265,9 @@ func NewAppender(ctx context.Context, d Driver, opts *AppendOptions) (*Appender,
 	}
 	// TODO(mhutchinson): move this into the decorators
 	a.Add = func(ctx context.Context, entry *Entry) IndexFuture {
+		if deadline, ok := ctx.Deadline(); ok {
+			appenderDeadlineRemaining.Record(ctx, time.Until(deadline).Milliseconds())
+		}
 		ctx, span := tracer.Start(ctx, "tessera.Appender.Add")
 		defer span.End()