[WIP] Introduce Bootstrap secrets

.github/workflows/jsonschema.yaml

@@ -32,4 +32,4 @@ jobs:
       - name: Verify secrets json schema
         run: |
           set -e
-          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done
+          for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done

.github/workflows/superlinter.yml

@@ -27,6 +27,8 @@ jobs:
           VALIDATE_ALL_CODEBASE: true
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Skip installed collection copies under .ansible/ (duplicate paths break PYTHON_MYPY and other tools).
+          FILTER_REGEX_EXCLUDE: '(^|/)\.ansible/'
           # These are the validation we disable atm
           VALIDATE_ANSIBLE: false
           VALIDATE_BIOME_FORMAT: false

Makefile

@@ -7,8 +7,9 @@ help: ## This help message
 
 .PHONY: super-linter
 super-linter: ## Runs super linter locally
-	rm -rf .mypy_cache
+	rm -rf .mypy_cache .ansible
 	podman run -e RUN_LOCAL=true -e USE_FIND_ALGORITHM=true	\
+					-e FILTER_REGEX_EXCLUDE='(^|/)\\.ansible/' \
 					-e VALIDATE_ANSIBLE=false \
 					-e VALIDATE_BASH=false \
 					-e VALIDATE_BIOME_FORMAT=false \
@@ -48,4 +49,4 @@ test: ansible-sanitytest ansible-unittest
 .PHONY: check-jsonschema
 check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones
 	set -e; \
-	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done
+	for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done

README.md

@@ -8,3 +8,92 @@ The main purpose of this collections are to:
 loading local secrets files into VP secrets stores.
 
 2. Help manage imperative and other utility functions of the cluster
+
+## Secrets loading
+
+Secrets are loaded from a **single primary** values-secret file (plus optional `values-secret.yaml.template` under the
+pattern tree as a last-resort discovery path). There are **no** separate `*-bootstrap.yaml` files or `VALUES_SECRET_BOOTSTRAP`
+paths; early cluster bootstrap uses **per-entry** `bootstrap` fields on v2 secrets in that same primary file.
+
+### Primary values-secret
+
+- **Backing store** comes from `values-global.yaml`: `.global.secretStore.backend` (default `vault`). That drives parsing
+  and whether secrets go to Vault or Kubernetes.
+- **Discovery order** when `VALUES_SECRET` is unset (first existing file wins):  
+  `~/.config/hybrid-cloud-patterns/values-secret-<pattern>.yaml`,  
+  `~/.config/validated-patterns/values-secret-<pattern>.yaml`,  
+  `~/values-secret-<pattern>.yaml`,  
+  `~/values-secret.yaml`,  
+  then `<pattern_dir>/values-secret.yaml.template`.
+- When `VALUES_SECRET` is set to an existing path, that file is used for the primary load.
+
+Files may be plain YAML or `ansible-vault` encrypted.
+
+### Per-secret `bootstrap` in v2 primary files
+
+On schema **2.0** primary values-secret files, each secret may set `bootstrap`:
+
+- **`bootstrap: true`** (or string equivalents such as `yes`, `both`) — the secret is included in the **early**
+  Kubernetes inject pass (`none` backend) and is **also** parsed in the **primary** pass into the configured backend
+  (Vault or Kubernetes as in `values-global.yaml`). It must not use `onMissingValue: generate` on any field (the early
+  pass cannot generate in Vault).
+- **`bootstrap: only`** (or `early`) — the secret is **only** in the early inject pass; the primary pass **omits** it.
+- **Unset / false** — normal primary-only secret.
+
+Invalid `bootstrap` scalars fail parsing with a clear error.
+
+Early inject runs **before** the primary backend load: during `playbooks/install.yml`, immediately after the
+pattern-install manifests are applied (`operator_deploy.yml`), then again inside `load_secrets` unless that early pass
+already completed (duplicate inject is skipped).
+
+### Playbooks and flows
+
+- **`playbooks/load_secrets.yml`**  
+  Respects `.global.secretLoader.disabled` in `values-global.yaml`. When enabled: `cluster_pre_check`, primary file
+  discovery, early Kubernetes inject for bootstrap-tagged v2 entries (when present), then parse and load the rest into
+  the configured backend.
+
+- **`playbooks/load_bootstrap_secrets.yml`**  
+  **Early bootstrap inject only**: `determine_pattern_dir`, `determine_pattern_name`, `pattern_settings`, then only the
+  Kubernetes inject for bootstrap-tagged secrets in the primary file (with retries). **Fails** if no primary file exists
+  or there are no bootstrap-tagged v2 entries. Does **not** read `secretLoader.disabled` or load into Vault / primary
+  backend. For the full early-then-primary flow, use `load_secrets.yml` (or `install.yml`).
+
+- **`playbooks/display_secrets_info.yml`**  
+  Loads and displays parsed secrets (using the backend from `values-global`). For v2 files with any bootstrap-tagged
+  entries, output is split into **`early_bootstrap_inject`** (none backend, early K8s view; includes `bootstrap: true`
+  and `bootstrap: only`) and **`primary_backend`** (configured backend; includes normal secrets and **`bootstrap: true`**
+  again so dual-mode entries appear in both groups). Otherwise a single parse is shown as before.
+
+Typical usage passes the pattern checkout as `pattern_dir` (for example `-e pattern_dir=/path/to/pattern`). If you omit
+it, the same resolution as `pattern_settings` applies: `PATTERN_DIR`, then `PWD`, then the `pwd` command.
+
+`playbooks/install.yml` imports `load_secrets.yml` after the pattern install playbook. When secret loading is enabled,
+early bootstrap inject from the primary file runs at the end of `operator_deploy.yml` (right after apply), then
+`load_secrets.yml` continues without repeating that inject when it already succeeded.
+
+### Early bootstrap inject retries
+
+Outer retries (parse plus Kubernetes apply) are controlled on the role defaults / extra-vars:
+
+- `vp_secrets_bootstrap_retry_max` (default `20`)
+- `vp_secrets_bootstrap_retry_delay` (seconds between attempts, default `30`)
+
+These apply to the early inject path inside `load_secrets` and to `load_bootstrap_secrets.yml`.
+
+Per-secret namespace readiness (before each `kubernetes.core.k8s` apply) uses role defaults on `k8s_secret_utils`:
+
+- `k8s_secret_namespace_check_retries` (default `5`) and `k8s_secret_namespace_check_delay` (seconds between attempts, default `45`).
+
+If the namespace still does not exist after those attempts, the inject fails and the **outer** retry re-runs parse plus
+all secret injections from the start.
+
+### Roles (implementation notes)
+
+- `roles/load_secrets/tasks/main.yml` implements the **combined** flow (early inject from primary file, then primary
+  backend load).
+- `roles/load_secrets/tasks/bootstrap_only.yml` is used when you invoke the `load_secrets` role with
+  `tasks_from: bootstrap_only.yml` (as `playbooks/load_bootstrap_secrets.yml` does).
+- `roles/find_vp_secrets` resolves the primary file (`tasks/main.yml`).
+- v2 parsing and phase filters (`bootstrap_only`, `exclude_bootstrap`, `all`) are implemented in
+  `plugins/module_utils/parse_secrets_v2.py` (single `bootstrap` normalizer: off / dual / early-only).

playbooks/determine_pattern_dir.yml

@@ -1,17 +1,24 @@
 ---
+# Resolves pattern_dir the same way as the pattern_settings role (extra-vars, PATTERN_DIR, PWD, pwd),
+# then fails if still unset. Used by display_secrets_info, load_values_global, load_bootstrap_secrets, etc.
 - name: Determine pattern dir
   hosts: localhost
   connection: local
   gather_facts: false
   become: false
-  vars:
-    pattern_dir: ''
   tasks:
-    - name: Fail if directory is not set
+    - name: Resolve pattern_dir from extra-vars, PATTERN_DIR, PWD, or pwd
+      ansible.builtin.include_role:
+        name: pattern_settings
+        tasks_from: resolve_overrides.yml
+
+    - name: Fail if pattern directory is not set after resolution
       ansible.builtin.fail:
-        msg: "pattern_dir variable must be set"
-      when: pattern_dir | length == 0
+        msg: >-
+          pattern_dir is not set. Pass -e pattern_dir=/path/to/pattern, export PATTERN_DIR to that path,
+          or run the playbook from the pattern directory so PWD is correct.
+      when: pattern_dir | default('') | string | trim | length == 0
 
     - name: Set pattern_dir fact for future plays
       ansible.builtin.set_fact:
-        pattern_dir: '{{ pattern_dir }}'
+        pattern_dir: "{{ pattern_dir | string | trim }}"

playbooks/display_secrets_info.yml

@@ -29,12 +29,70 @@
       ansible.builtin.set_fact:
         secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}"
 
-    - name: Parse secrets data
+    - name: Detect inline bootstrap secrets in primary v2 file
+      ansible.builtin.set_fact:
+        _vp_has_inline_bootstrap_secrets: >-
+          {{
+            (secrets_yaml.version | default('2.0')) is version('2.0', '>=')
+            and (
+              (secrets_yaml.secrets | default([])
+              | selectattr('bootstrap', 'defined')
+              | selectattr('bootstrap')
+              | list
+              | length) > 0
+            )
+          }}
+
+    - name: Parse secrets data (v2 with bootstrap — two display groups)
+      when: _vp_has_inline_bootstrap_secrets | bool
+      block:
+        - name: Parse early-bootstrap inject portion for display (none backend)
+          no_log: '{{ hide_sensitive_output }}'
+          parse_secrets_info:
+            values_secrets_plaintext: "{{ values_secrets_data }}"
+            secrets_backing_store: none
+            secrets_parse_filter: bootstrap_only
+          register: _display_bootstrap_parse
+
+        - name: Parse primary-backend portion for display (configured backend)
+          no_log: '{{ hide_sensitive_output }}'
+          parse_secrets_info:
+            values_secrets_plaintext: "{{ values_secrets_data }}"
+            secrets_backing_store: "{{ secrets_backing_store }}"
+            secrets_parse_filter: exclude_bootstrap
+          register: _display_primary_parse
+
+        - name: Build two-group secrets display (dual bootstrap entries appear in both)
+          ansible.builtin.set_fact:
+            secrets_results:
+              early_bootstrap_inject:
+                parsed_secrets: "{{ _display_bootstrap_parse.parsed_secrets }}"
+                kubernetes_secret_objects: "{{ _display_bootstrap_parse.kubernetes_secret_objects }}"
+                vault_policies: "{{ _display_bootstrap_parse.vault_policies | default({}) }}"
+                unique_vault_prefixes: "{{ _display_bootstrap_parse.unique_vault_prefixes | default([]) }}"
+                backing_store: none
+              primary_backend:
+                parsed_secrets: "{{ _display_primary_parse.parsed_secrets }}"
+                kubernetes_secret_objects: "{{ _display_primary_parse.kubernetes_secret_objects }}"
+                vault_policies: "{{ _display_primary_parse.vault_policies | default({}) }}"
+                secret_store_namespace: "{{ _display_primary_parse.secret_store_namespace }}"
+                unique_vault_prefixes: "{{ _display_primary_parse.unique_vault_prefixes | default([]) }}"
+                secrets_backing_store: "{{ secrets_backing_store }}"
+
+    # Do not register: secrets_results here — a skipped task still overwrites the register
+    # and would wipe the two-group set_fact when bootstrap secrets are present.
+    - name: Parse secrets data (single phase)
+      when: not (_vp_has_inline_bootstrap_secrets | bool)
       no_log: '{{ hide_sensitive_output }}'
       parse_secrets_info:
         values_secrets_plaintext: "{{ values_secrets_data }}"
         secrets_backing_store: "{{ secrets_backing_store }}"
-      register: secrets_results
+      register: _display_single_phase_parse
+
+    - name: Set secrets_results from single-phase parse
+      when: not (_vp_has_inline_bootstrap_secrets | bool)
+      ansible.builtin.set_fact:
+        secrets_results: "{{ _display_single_phase_parse }}"
 
     - name: Display secrets data
       ansible.builtin.debug:

playbooks/load_bootstrap_secrets.yml

@@ -0,0 +1,25 @@
+---
+# Inject only bootstrap-tagged secrets from the primary values-secret file (none backend, with retries).
+# Does not load secrets into Vault or the primary Kubernetes backend. Does not honor
+# secretLoader.disabled from values-global. Fails if no primary file exists or there are no
+# bootstrap-tagged v2 entries.
+# Optional extra-vars: vp_secrets_bootstrap_retry_max, vp_secrets_bootstrap_retry_delay (seconds).
+- name: Determine pattern directory
+  ansible.builtin.import_playbook: ./determine_pattern_dir.yml
+
+- name: Determine pattern name
+  ansible.builtin.import_playbook: ./determine_pattern_name.yml
+
+- name: Load bootstrap secrets
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  become: false
+  roles:
+    - pattern_settings
+
+  tasks:
+    - name: Run bootstrap-only secrets load
+      ansible.builtin.include_role:
+        name: load_secrets
+        tasks_from: bootstrap_only.yml

playbooks/operator_deploy.yml

@@ -51,3 +51,29 @@
             msg: |
               Failed to install pattern after 10 retries.
               Error: {{ _apply.error | default(_apply.msg) | default('Unknown error') }}
+
+    # Bootstrap-tagged secrets in the primary values-secret file run immediately after apply
+    # (before full load_secrets and Argo health wait).
+    - name: Evaluate secret loader setting for bootstrap timing
+      ansible.builtin.set_fact:
+        secret_loader_disabled: "{{ values_global.global.secretLoader.disabled | default(false) | bool }}"
+
+    - name: Run cluster pre-check and early bootstrap from primary file after pattern apply
+      when: not secret_loader_disabled
+      block:
+        - name: Run cluster pre-check before bootstrap
+          ansible.builtin.include_role:
+            name: cluster_pre_check
+
+        - name: Remember cluster pre-check completed (avoid duplicate in load_secrets)
+          ansible.builtin.set_fact:
+            vp_cluster_pre_check_done: true
+
+        - name: Early inject of bootstrap-tagged secrets from primary values-secret
+          ansible.builtin.include_role:
+            name: load_secrets
+            tasks_from: early_bootstrap_from_primary.yml
+
+        - name: Remember early bootstrap inject for duplicate skip in load_secrets
+          ansible.builtin.set_fact:
+            vp_early_primary_bootstrap_done: "{{ vp_early_primary_file_loaded | default(false) | bool }}"

playbooks/process_secrets.yml

@@ -18,10 +18,10 @@
         - find_vp_secrets
 
     # find_vp_secrets will return a plaintext data structure called values_secrets_data
-    # This will allow us to determine schema version and which backend to use
-    - name: Determine how to load secrets
-      ansible.builtin.set_fact:
-        secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}"
+    - name: Early inject of bootstrap-tagged secrets from primary file (v2)
+      ansible.builtin.include_role:
+        name: load_secrets
+        tasks_from: inject_early_bootstrap_primary_entries.yml
 
     - name: Parse secrets data
       no_log: '{{ hide_sensitive_output | default(true) }}'