refactor client error logging, change not_saved metric to no_attestation, add rate_limited metric

Author: piceri

README.md

@@ -184,6 +184,10 @@ The metrics exposed beyond the default Prometheus metrics are:
   outgoing HTTP POST to upload the deployment record.
 * `deptracker_post_record_ok`: the number of successful deployment
   record uploads.
+* `deptracker_post_record_rate_limited`: the number of post attempts 
+  that were rate limited.
+* `deptracker_post_record_no_attestation`: the number of successful 
+  posts for container digest with no matching attestation for the org.
 * `deptracker_post_record_soft_fail`: the number of recoverable failed
   attempts to upload the deployment record.
 * `deptracker_post_record_hard_fail`: the number of failures to

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect

pkg/deploymentrecord/client.go

@@ -166,7 +166,10 @@ type NoArtifactError struct {
 }
 
 func (n *NoArtifactError) Error() string {
-	return fmt.Sprintf("client_error: %s", n.err.Error())
+	if n.err == nil {
+		return "no artifact found"
+	}
+	return fmt.Sprintf("no artifact found: %s", n.err.Error())
 }
 
 func (n *NoArtifactError) Unwrap() error {
@@ -262,35 +265,39 @@ func (c *Client) PostOne(ctx context.Context, record *DeploymentRecord) error {
 		}
 
 		// Drain and close response body to enable connection reuse by reading body for error logging
-		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
 		_, _ = io.Copy(io.Discard, resp.Body)
 		_ = resp.Body.Close()
 
 		switch {
-		case resp.StatusCode == 429:
-			// Retry with backoff
-			dtmetrics.PostDeploymentRecordSoftFail.Inc()
-			slog.Debug("rate limited, retrying",
-				"attempt", attempt,
-				"status_code", resp.StatusCode,
-			)
-			lastErr = fmt.Errorf("rate limited, attempt %d", attempt)
-			continue
-		case resp.StatusCode == 404:
+		case resp.StatusCode == 404 && bytes.Contains(respBody, []byte("no artifacts found")):
 			// No artifact found
-			dtmetrics.PostDeploymentRecordNotSaved.Inc()
+			dtmetrics.PostDeploymentRecordNoAttestation.Inc()
 			slog.Debug("no artifact attestation found, no record created",
 				"attempt", attempt,
 				"status_code", resp.StatusCode,
 			)
-			return &NoArtifactError{err: fmt.Errorf("no artifact found")}
+			return &NoArtifactError{}
 		case resp.StatusCode >= 400 && resp.StatusCode < 500:
+			if resp.Header.Get("retry-after") != "" || resp.Header.Get("x-ratelimit-remaining") == "0" {
+				// rate limited — retry with backoff
+				// Could be 403 or 429
+				dtmetrics.PostDeploymentRecordRateLimited.Inc()
+				slog.Warn("rate limited, retrying",
+					"attempt", attempt,
+					"status_code", resp.StatusCode,
+					"retry_after", resp.Header.Get("Retry-After"),
+					"resp_msg", string(respBody),
+				)
+				lastErr = fmt.Errorf("rate limited, attempt %d", attempt)
+				continue
+			}
 			// Don't retry non rate limiting client errors
 			dtmetrics.PostDeploymentRecordClientError.Inc()
 			slog.Warn("client error, aborting",
 				"attempt", attempt,
 				"status_code", resp.StatusCode,
-				"resp_msg", string(body),
+				"resp_msg", string(respBody),
 			)
 			return &ClientError{err: fmt.Errorf("unexpected client err with status code %d", resp.StatusCode)}
 		default:
@@ -299,7 +306,7 @@ func (c *Client) PostOne(ctx context.Context, record *DeploymentRecord) error {
 			slog.Debug("retriable error",
 				"attempt", attempt,
 				"status_code", resp.StatusCode,
-				"resp_msg", string(body),
+				"resp_msg", string(respBody),
 			)
 			lastErr = fmt.Errorf("server error, attempt %d", attempt)
 		}