fix: address PR comments

Author: ajbeattie

internal/controller/reporting.go

@@ -183,6 +183,16 @@ func (c *Controller) processSyncEvents(ctx context.Context, syncClusterPods []an
 		"errors", len(jobStatus.Errors),
 	)
 
+	// If the job failed, don't populate observedDeployments — records may not
+	// have been created, and suppressing re-posts would delay self-healing.
+	// Only populate unknownArtifacts from errors so we avoid redundant 404s.
+	if jobStatus.Status == "failed" {
+		slog.Warn("Cluster job failed, skipping observedDeployments cache fill",
+			"job_id", jobResp.JobID,
+		)
+		return nil
+	}
+
 	c.fillCachesFromJobResult(syncRecords, jobResp, jobStatus)
 	return nil
 }

internal/controller/reporting_test.go

@@ -331,6 +331,35 @@ func TestProcessSyncEvents_JobWaitFailed(t *testing.T) {
 	assert.True(t, exists, "observedDeployments should be populated from submitted records on wait failure")
 }
 
+func TestProcessSyncEvents_JobStatusFailed(t *testing.T) {
+	t.Parallel()
+	digest := "sha256:abc123"
+	poster := &mockPoster{
+		jobResp: &deploymentrecord.JobResponse{JobID: 42},
+		jobStatus: &deploymentrecord.JobStatus{
+			JobID:  42,
+			Status: "failed",
+			Errors: []deploymentrecord.JobError{
+				{Name: "nginx", Cause: "error"},
+			},
+		},
+	}
+	ctrl := newTestController(poster)
+	ctrl.workloadResolver = &mockResolver{name: "test-deploy"}
+	pod := makeTestPod("app", "test-deploy-abc123", digest, "ReplicaSet")
+
+	err := ctrl.processSyncEvents(context.Background(), []any{pod})
+	require.NoError(t, err, "failed job should not block startup")
+	assert.Equal(t, 1, poster.getCreateClusterJobCalls())
+	assert.Equal(t, 1, poster.getWaitForClusterJobCalls())
+
+	// observedDeployments should NOT be populated — records may not have been
+	// created, and suppressing re-posts would delay self-healing.
+	cacheKey := getCacheKey(EventCreated, "default/test-deploy/app", digest)
+	_, exists := ctrl.observedDeployments.Get(cacheKey)
+	assert.False(t, exists, "observedDeployments should not be populated when job failed")
+}
+
 func TestMakeSyncRecords_TerminalJobPodIncluded(t *testing.T) {
 	t.Parallel()
 	digest := "sha256:terminal-job-digest"

pkg/deploymentrecord/client.go

@@ -504,7 +504,7 @@ func (c *Client) doWithRetry(ctx context.Context, method, targetURL string, body
 				"status_code", resp.StatusCode,
 				"resp_msg", string(respBody),
 			)
-			return nil, resp.StatusCode, &ClientError{err: fmt.Errorf("unexpected client err with status code %d", resp.StatusCode)}
+			return respBody, resp.StatusCode, &ClientError{err: fmt.Errorf("unexpected client err with status code %d", resp.StatusCode)}
 		default:
 			// Retry with backoff
 			dtmetrics.PostDeploymentRecordSoftFail.Inc()