fix(monitoring): stop tracking the polarsignals-cloud Secret in ArgoCD (#476)

The empty-shell pattern (commit the object with no data, live-patch
the token) kept breaking: the token went missing twice in a row right
after unrelated syncs of this same Application, each time taking down
remote-write to Polar Signals Cloud until manually recreated. Rather
than chase the exact SSA mechanism, remove the object from git
entirely so ArgoCD never applies or diffs it at all — this repo tracks
resources by the argocd.argoproj.io/tracking-id annotation, so an
object without one is invisible to sync and prune alike. The
Prometheus CR still references the Secret by name; only the object
that provisions it is gone from here.

Author: metalmatze

monitoring/README.md

@@ -51,13 +51,22 @@ kubectl --namespace=parca-analytics patch middleware psc-remote-write-headers \
 ## Sending parca-analytics data to Polar Signals Cloud via remoteWrite
 
 The parca-analytics Prometheus has a second `remoteWrite` target pointing
-directly at Polar Signals Cloud. The token is not committed to git — the
-`polarsignals-cloud` Secret is committed with no `data` at all, matching the
-`objectStorageSecret` pattern above. Once you have a Polar Signals Cloud API
-token, patch it in:
+directly at Polar Signals Cloud. The token is not committed to git, and unlike
+`objectStorageSecret`, the `polarsignals-cloud` Secret isn't declared in git
+at all — it's created directly in the cluster and left untracked by ArgoCD,
+so a sync can never prune or reset it. Once you have a Polar Signals Cloud API
+token, create it:
 
 ```bash
 kubectl --namespace=parca-analytics create secret generic polarsignals-cloud \
-  --from-literal=token=<token> --dry-run=client -o yaml \
-  | kubectl apply -f -
+  --from-literal=token=<token>
+```
+
+If the Secret already exists with ArgoCD's `argocd.argoproj.io/tracking-id`
+annotation on it (e.g. left over from before this Secret was untracked),
+remove that annotation so `prune: true` syncs leave it alone:
+
+```bash
+kubectl --namespace=parca-analytics annotate secret polarsignals-cloud \
+  argocd.argoproj.io/tracking-id-
 ```

monitoring/environments/scaleway-parca-demo/main.jsonnet

@@ -327,7 +327,11 @@ local prometheuses = [
         remoteWrite: [{
           url: 'https://api.polarsignals.com/api/v1/write',
           authorization: {
-            credentials: { name: p.polarSignalsCloudSecret.metadata.name, key: 'token' },
+            // Deliberately not the polarSignalsCloudSecret object it once was: this
+            // Secret is created directly in the cluster, untracked by ArgoCD (see
+            // monitoring/README.md), so nothing ever prunes or resyncs its token
+            // away. Referencing it by name here is the only coupling left.
+            credentials: { name: 'polarsignals-cloud', key: 'token' },
           },
           headers: {
             projectID: '5a755043-1fb8-48fd-a2c8-2787498ec59d',
@@ -379,16 +383,6 @@ local prometheuses = [
       },
     },
 
-    polarSignalsCloudSecret: {
-      apiVersion: 'v1',
-      kind: 'Secret',
-      metadata: {
-        name: 'polarsignals-cloud',
-        namespace: p._config.namespace,
-        labels: p._config.commonLabels,
-      },
-    },
-
     polarSignalsCloudRemoteWriteSLO: {
       apiVersion: 'pyrra.dev/v1alpha1',
       kind: 'ServiceLevelObjective',