Revert "Merge branch 'bootstrap_secrets_single_file' into feature/sscsi-vp-proxy-cluster-ca-chart"

This reverts commit 8bbdc7b, reversing
changes made to 74657c3.

Author: Martin Jackson

.github/workflows/jsonschema.yaml

@@ -32,4 +32,4 @@ jobs:
       - name: Verify secrets json schema
         run: |
           set -e
-          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done
+          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done

.github/workflows/superlinter.yml

@@ -27,8 +27,6 @@ jobs:
           VALIDATE_ALL_CODEBASE: true
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Skip installed collection copies under .ansible/ (duplicate paths break PYTHON_MYPY and other tools).
-          FILTER_REGEX_EXCLUDE: '(^|/)\.ansible/'
           # These are the validation we disable atm
           VALIDATE_ANSIBLE: false
           VALIDATE_BIOME_FORMAT: false

.github/workflows/trigger-utility-imperative-container-builds.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Generate GitHub App token
         id: generate-token
-        uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
+        uses: actions/create-github-app-token@f8d387b68d61c58ab83c6c016672934102569859 # v3.0.0
         with:
           app-id: ${{ secrets.GH_WORKFLOW_AUTOMATION_CLIENT_ID }}
           private-key: ${{ secrets.GH_WORKFLOW_AUTOMATION_PRIVATE_KEY }}

Makefile

@@ -7,9 +7,9 @@ help: ## This help message
 
 .PHONY: super-linter
 super-linter: ## Runs super linter locally
-	rm -rf .mypy_cache .ansible
+	rm -rf .mypy_cache
+	rm -rf .ansible
 	podman run -e RUN_LOCAL=true -e USE_FIND_ALGORITHM=true	\
-					-e FILTER_REGEX_EXCLUDE='(^|/)\\.ansible/' \
 					-e VALIDATE_ANSIBLE=false \
 					-e VALIDATE_BASH=false \
 					-e VALIDATE_BIOME_FORMAT=false \
@@ -49,4 +49,4 @@ test: ansible-sanitytest ansible-unittest
 .PHONY: check-jsonschema
 check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones
 	set -e; \
-	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done
+	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done

README.md

@@ -9,94 +9,84 @@ loading local secrets files into VP secrets stores.
 
 2. Help manage imperative and other utility functions of the cluster
 
-## Secrets loading
-
-Secrets are loaded from a **single primary** values-secret file (plus optional `values-secret.yaml.template` under the
-pattern tree as a last-resort discovery path). There are **no** separate `*-bootstrap.yaml` files or `VALUES_SECRET_BOOTSTRAP`
-paths; early cluster bootstrap uses **per-entry** `bootstrap` fields on v2 secrets in that same primary file.
-
-### Primary values-secret
-
-- **Backing store** comes from `values-global.yaml`: `.global.secretStore.backend` (default `vault`). That drives parsing
-  and whether secrets go to Vault or Kubernetes.
-- **Discovery order** when `VALUES_SECRET` is unset (first existing file wins):  
-  `~/.config/hybrid-cloud-patterns/values-secret-<pattern>.yaml`,  
-  `~/.config/validated-patterns/values-secret-<pattern>.yaml`,  
-  `~/values-secret-<pattern>.yaml`,  
-  `~/values-secret.yaml`,  
-  then `<pattern_dir>/values-secret.yaml.template`.
-- When `VALUES_SECRET` is set to an existing path, that file is used for the primary load.
-
-Files may be plain YAML or `ansible-vault` encrypted.
-
-### Per-secret `bootstrap` in v2 primary files
-
-On schema **2.0** primary values-secret files, each secret may set `bootstrap`:
-
-- **`bootstrap: true`** (or string equivalents such as `yes`, `both`) — the secret is included in the **early**
-  Kubernetes inject pass (`none` backend) and is **also** parsed in the **primary** pass into the configured backend
-  (Vault or Kubernetes as in `values-global.yaml`). It must not use `onMissingValue: generate` on any field (the early
-  pass cannot generate in Vault).
-- **`bootstrap: only`** (or `early`) — the secret is **only** in the early inject pass; the primary pass **omits** it.
-- **Unset / false** — normal primary-only secret.
-
-Invalid `bootstrap` scalars fail parsing with a clear error.
-
-Early inject runs **before** the primary backend load: during `playbooks/install.yml`, immediately after the
-pattern-install manifests are applied (`operator_deploy.yml`), then again inside `load_secrets` unless that early pass
-already completed (duplicate inject is skipped).
-
-### Playbooks and flows
-
-- **`playbooks/load_secrets.yml`**  
-  Respects `.global.secretLoader.disabled` in `values-global.yaml`. When enabled: `cluster_pre_check`, primary file
-  discovery, early Kubernetes inject for bootstrap-tagged v2 entries (when present), then parse and load the rest into
-  the configured backend.
-
-- **`playbooks/load_bootstrap_secrets.yml`**  
-  Convenience wrapper: `determine_pattern_dir`, `determine_pattern_name`, then imports `load_secrets.yml` (same behavior
-  as install).
-
-- **`playbooks/load_bootstrap_secrets_only.yml`**  
-  **Early bootstrap inject only**: same pattern discovery plays and `pattern_settings`, then only the Kubernetes inject
-  for bootstrap-tagged secrets in the primary file (with retries). **Fails** if no primary file exists or there are no
-  bootstrap-tagged v2 entries. Does **not** read `secretLoader.disabled` or load into Vault / primary backend.
-
-- **`playbooks/display_secrets_info.yml`**  
-  Loads and displays parsed secrets (using the backend from `values-global`). For v2 files with any bootstrap-tagged
-  entries, output is split into **`early_bootstrap_inject`** (none backend, early K8s view; includes `bootstrap: true`
-  and `bootstrap: only`) and **`primary_backend`** (configured backend; includes normal secrets and **`bootstrap: true`**
-  again so dual-mode entries appear in both groups). Otherwise a single parse is shown as before.
-
-Typical usage passes the pattern checkout as `pattern_dir` (for example `-e pattern_dir=/path/to/pattern`). If you omit
-it, the same resolution as `pattern_settings` applies: `PATTERN_DIR`, then `PWD`, then the `pwd` command.
-
-`playbooks/install.yml` imports `load_secrets.yml` after the pattern install playbook. When secret loading is enabled,
-early bootstrap inject from the primary file runs at the end of `operator_deploy.yml` (right after apply), then
-`load_secrets.yml` continues without repeating that inject when it already succeeded.
-
-### Early bootstrap inject retries
-
-Outer retries (parse plus Kubernetes apply) are controlled on the role defaults / extra-vars:
-
-- `vp_secrets_bootstrap_retry_max` (default `20`)
-- `vp_secrets_bootstrap_retry_delay` (seconds between attempts, default `30`)
-
-These apply to the early inject path inside `load_secrets` and to `load_bootstrap_secrets_only.yml`.
-
-Per-secret namespace readiness (before each `kubernetes.core.k8s` apply) uses role defaults on `k8s_secret_utils`:
-
-- `k8s_secret_namespace_check_retries` (default `5`) and `k8s_secret_namespace_check_delay` (seconds between attempts, default `45`).
-
-If the namespace still does not exist after those attempts, the inject fails and the **outer** retry re-runs parse plus
-all secret injections from the start.
-
-### Roles (implementation notes)
-
-- `roles/load_secrets/tasks/main.yml` implements the **combined** flow (early inject from primary file, then primary
-  backend load).
-- `roles/load_secrets/tasks/bootstrap_only.yml` is used only when you invoke the `load_secrets` role with
-  `tasks_from: bootstrap_only.yml` (as `load_bootstrap_secrets_only.yml` does).
-- `roles/find_vp_secrets` resolves the primary file (`tasks/main.yml`).
-- v2 parsing and phase filters (`bootstrap_only`, `exclude_bootstrap`, `all`) are implemented in
-  `plugins/module_utils/parse_secrets_v2.py` (single `bootstrap` normalizer: off / dual / early-only).
+## SS CSI workload auth notes
+
+SS CSI task files live in **`roles/vault_utils/tasks/ss_csi/`**; paths below match **`include_tasks`** from the role (**`ss_csi/<file>`** relative to **`tasks/`**).
+
+`vault_utils` can read `ssCsiWorkloadAuth` entries from clustergroup values and
+create Vault Kubernetes auth roles for hub and spoke workloads.
+
+### Parsing (load YAML)
+
+With **`vault_ss_csi_aggregate_clustergroup_sources`** true (default), SS CSI
+uses the **`clustergroup_discovery`** role to determine stems: **main** from
+`values-global.yaml`, then **managed** names from `clusterGroup.managedClusterGroups`
+in the main `values-<main>.yaml|yml`. For **each** stem it loads a document from
+the in-cluster **`ConfigMap` `values-<stem>`** (namespace
+`openshift-gitops` by default), then falls back to **`pattern_dir/values-<stem>.yaml|yml`**
+when enabled. ConfigMap data keys follow **`vault_ss_csi_clustergroup_configmap_key`**
+and **`vault_ss_csi_clustergroup_configmap_key_candidates`**. Each document must
+include **`clusterGroup`**. Stems are merged in **`clustergroup_load_order`**
+(main first, then managed stems sorted) so later sources override duplicate
+`clusterGroup.applications` keys. Set **`vault_ss_csi_aggregate_clustergroup_sources`**
+to false to load only the **main** document (legacy: single ConfigMap or
+`values-<main>.yaml`).
+
+### Extraction (find `ssCsiWorkloadAuth`)
+
+The role builds **`_vault_ss_csi_apps_by_stem`** (per-stem `clusterGroup.applications`)
+and a merged **`clusterGroup.managedClusterGroups`**. It collects:
+
+- **`clusterGroup.applications.*.ssCsiWorkloadAuth`** — per stem; omit **`cluster`**
+  in values: the **main** stem resolves to **hub**; **managed** stems resolve to
+  that **stem name** so entries under `values-<managed>.yaml` stay spoke-scoped.
+- **`clusterGroup.managedClusterGroups.*.applications.*.ssCsiWorkloadAuth`** —
+  from the merged map; omit **`cluster`** and the row targets that managed group
+  (**`name`**, else the group map key).
+
+### Projection (Vault roles)
+
+Rows are appended to **`_ss_csi_all_entries`**, split into hub vs spoke using
+the computed **`cluster`** field (from stem or managed group when omitted in YAML), then **hub** identities get Vault Kubernetes
+auth roles via **`ss_csi/vault_ss_csi_apply_one_hub_sscsi_role.yaml`**. Spoke rows are
+normalized to **`vault_path`** later in the play (**`ss_csi/vault_ss_csi_normalize_spoke_entries_to_vault_path.yaml`**
+during **`vault_spokes_init`**) and roles are written on each spoke mount
+(**`ss_csi/vault_ss_csi_apply_one_spoke_sscsi_role.yaml`**). Role names use
+**`<mount>-sscsi-<slug>`**; slugs come from **`ss_csi/vault_ss_csi_compute_role_slug.yaml`**.
+
+To **inspect** stems and files locally, run **`playbooks/list_clustergroups.yml`**
+or **`playbooks/parse_clustergroup_values.yml`** (see **`roles/clustergroup_discovery/README.md`**).
+
+At the application level (`clusterGroup.applications.<app>`), the relevant
+inputs are:
+
+- `ssCsiWorkloadAuth` (list)
+- `ssCsiWorkloadAuth[].serviceAccount` (required)
+- `ssCsiWorkloadAuth[].namespace` (optional)
+- Omit **`cluster`** in pattern YAML; hub vs spoke comes from **which file or
+  `managedClusterGroups` branch** defines the list (see extraction above). Spoke
+  handling still normalizes to **`vault_path`** (full DNS), same as External Secrets.
+- `ssCsiWorkloadAuth[].roleSlug` / `role_slug` (optional): suffix only; Vault
+  role is **`<mount>-sscsi-<slug>`** where **`<mount>`** is hub **`hub`** (or
+  configured hub path) or the spoke **`vault_path`**. When using the
+  **vp-sscsi-spc** chart, `spec.parameters.roleName` uses the same **mount**
+  as `vaultKubernetesMountPath` (typically **`global.clusterDomain`** on
+  spokes), not a short clustergroup label.
+- application `namespace` (optional default for entry namespace)
+
+CA material management for SS CSI is not handled in this collection anymore.
+Provide CA distribution using a separate chart or platform mechanism.
+
+For the complete flow and task ordering, see
+`secrets-initialization-and-vault-unseal.md`.
+
+## Pattern repository directory (`pattern_dir`)
+
+Playbooks need the path to your pattern Git checkout (where `values-global.yaml`
+and related files live). Resolution order: extra var `pattern_dir`, environment
+variable `PATTERN_DIR`, then `PWD` and `pwd`.
+
+When running from the imperative container or another fixed working directory,
+pass the repository root explicitly, for example `-e pattern_dir=/git/repo` (or add
+equivalent extra vars via `clusterGroup.imperative.extraPlaybookArgs` in the
+clustergroup chart).

playbooks/determine_pattern_dir.yml

@@ -1,24 +1,17 @@
 ---
-# Resolves pattern_dir the same way as the pattern_settings role (extra-vars, PATTERN_DIR, PWD, pwd),
-# then fails if still unset. Used by display_secrets_info, load_values_global, load_bootstrap_secrets*, etc.
 - name: Determine pattern dir
   hosts: localhost
   connection: local
   gather_facts: false
   become: false
+  vars:
+    pattern_dir: ''
   tasks:
-    - name: Resolve pattern_dir from extra-vars, PATTERN_DIR, PWD, or pwd
-      ansible.builtin.include_role:
-        name: pattern_settings
-        tasks_from: resolve_overrides.yml
-
-    - name: Fail if pattern directory is not set after resolution
+    - name: Fail if directory is not set
       ansible.builtin.fail:
-        msg: >-
-          pattern_dir is not set. Pass -e pattern_dir=/path/to/pattern, export PATTERN_DIR to that path,
-          or run the playbook from the pattern directory so PWD is correct.
-      when: pattern_dir | default('') | string | trim | length == 0
+        msg: "pattern_dir variable must be set"
+      when: pattern_dir | length == 0
 
     - name: Set pattern_dir fact for future plays
       ansible.builtin.set_fact:
-        pattern_dir: "{{ pattern_dir | string | trim }}"
+        pattern_dir: '{{ pattern_dir }}'

playbooks/display_secrets_info.yml

@@ -29,70 +29,12 @@
       ansible.builtin.set_fact:
         secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}"
 
-    - name: Detect inline bootstrap secrets in primary v2 file
-      ansible.builtin.set_fact:
-        _vp_has_inline_bootstrap_secrets: >-
-          {{
-            (secrets_yaml.version | default('2.0')) is version('2.0', '>=')
-            and (
-              (secrets_yaml.secrets | default([])
-              | selectattr('bootstrap', 'defined')
-              | selectattr('bootstrap')
-              | list
-              | length) > 0
-            )
-          }}
-
-    - name: Parse secrets data (v2 with bootstrap — two display groups)
-      when: _vp_has_inline_bootstrap_secrets | bool
-      block:
-        - name: Parse early-bootstrap inject portion for display (none backend)
-          no_log: '{{ hide_sensitive_output }}'
-          parse_secrets_info:
-            values_secrets_plaintext: "{{ values_secrets_data }}"
-            secrets_backing_store: none
-            secrets_parse_filter: bootstrap_only
-          register: _display_bootstrap_parse
-
-        - name: Parse primary-backend portion for display (configured backend)
-          no_log: '{{ hide_sensitive_output }}'
-          parse_secrets_info:
-            values_secrets_plaintext: "{{ values_secrets_data }}"
-            secrets_backing_store: "{{ secrets_backing_store }}"
-            secrets_parse_filter: exclude_bootstrap
-          register: _display_primary_parse
-
-        - name: Build two-group secrets display (dual bootstrap entries appear in both)
-          ansible.builtin.set_fact:
-            secrets_results:
-              early_bootstrap_inject:
-                parsed_secrets: "{{ _display_bootstrap_parse.parsed_secrets }}"
-                kubernetes_secret_objects: "{{ _display_bootstrap_parse.kubernetes_secret_objects }}"
-                vault_policies: "{{ _display_bootstrap_parse.vault_policies | default({}) }}"
-                unique_vault_prefixes: "{{ _display_bootstrap_parse.unique_vault_prefixes | default([]) }}"
-                backing_store: none
-              primary_backend:
-                parsed_secrets: "{{ _display_primary_parse.parsed_secrets }}"
-                kubernetes_secret_objects: "{{ _display_primary_parse.kubernetes_secret_objects }}"
-                vault_policies: "{{ _display_primary_parse.vault_policies | default({}) }}"
-                secret_store_namespace: "{{ _display_primary_parse.secret_store_namespace }}"
-                unique_vault_prefixes: "{{ _display_primary_parse.unique_vault_prefixes | default([]) }}"
-                secrets_backing_store: "{{ secrets_backing_store }}"
-
-    # Do not register: secrets_results here — a skipped task still overwrites the register
-    # and would wipe the two-group set_fact when bootstrap secrets are present.
-    - name: Parse secrets data (single phase)
-      when: not (_vp_has_inline_bootstrap_secrets | bool)
+    - name: Parse secrets data
       no_log: '{{ hide_sensitive_output }}'
       parse_secrets_info:
         values_secrets_plaintext: "{{ values_secrets_data }}"
         secrets_backing_store: "{{ secrets_backing_store }}"
-      register: _display_single_phase_parse
-
-    - name: Set secrets_results from single-phase parse
-      when: not (_vp_has_inline_bootstrap_secrets | bool)
-      ansible.builtin.set_fact:
-        secrets_results: "{{ _display_single_phase_parse }}"
+      register: secrets_results
 
     - name: Display secrets data
       ansible.builtin.debug: