[reproducer] Fix PreMetal handling of extra_variable_files

PreMetal provisions hosts from the main branch, generating
reproducer-variables.yml with base scenario vars. During the actual
Zuul job, overwrite_zuul_vars.yml only merges zuul_vars.yaml into
reproducer-variables.yml, ignoring extra_variable_files entirely.

Additionally, the slurp + copy-content pattern for transferring
reproducer-variables.yml triggers Jinja2 evaluation on the file
content, breaking when the bootstrap bakes raw templates into
nested structures.

Signed-off-by: Samuel Rac <srac@redhat.com>

Author: srac0

roles/reproducer/tasks/overwrite_zuul_vars.yml

@@ -19,18 +19,12 @@
     temp_reproducer_var_path: /tmp/reproducer-variables.yml
     temp_merged_reproducer_var_path: /tmp/merged-reproducer-variables.yml
   block:
-    - name: Slurp reproducer-variables.yml to hypervisor
-      ansible.builtin.slurp:
+    - name: Fetch reproducer-variables.yml from controller-0
+      ansible.builtin.fetch:
         src: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml"
-      register: reproducer_original
-      delegate_to: controller-0
-      no_log: "{{ cifmw_nolog | default(true) | bool }}"
-
-    - name: Create temp file reproducer-variables.yml on hypervisor
-      ansible.builtin.copy:
-        content: "{{ reproducer_original.content | b64decode }}"
         dest: "{{ temp_reproducer_var_path }}"
-        mode: "0664"
+        flat: true
+      delegate_to: controller-0
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
     - name: Copy merge yamls script
@@ -48,15 +42,48 @@
         {{ temp_merged_reproducer_var_path }}
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
-    - name: Slurp merged reproducer-variables.yml from hypervisor
-      ansible.builtin.slurp:
-        src: "{{ temp_merged_reproducer_var_path }}"
-      register: merged_reproducer_slurp
+    - name: Merge scenario variable files into reproducer-variables.yml
+      when: variable_files_dirs is defined
+      block:
+        - name: Find scenario variable files
+          ansible.builtin.find:
+            paths: "{{ variable_files_dirs }}"
+            patterns: "*.yaml,*.yml"
+            recurse: false
+          register: _scenario_var_files
+
+        - name: Merge each scenario variable file
+          ansible.builtin.shell: >
+            python3 /usr/local/bin/merge_yaml_override
+            {{ temp_merged_reproducer_var_path }}
+            {{ scenario_var_file.path }} >
+            {{ temp_merged_reproducer_var_path }}.tmp &&
+            mv {{ temp_merged_reproducer_var_path }}.tmp
+            {{ temp_merged_reproducer_var_path }}
+          loop: "{{ _scenario_var_files.files | sort(attribute='path') }}"
+          loop_control:
+            loop_var: scenario_var_file
+            label: "{{ scenario_var_file.path | basename }}"
+          no_log: "{{ cifmw_nolog | default(true) | bool }}"
+
+    - name: Merge extra variable files into reproducer-variables.yml
+      when: extra_variable_files is defined
+      ansible.builtin.shell: >
+        python3 /usr/local/bin/merge_yaml_override
+        {{ temp_merged_reproducer_var_path }}
+        {{ extra_variable_file }} >
+        {{ temp_merged_reproducer_var_path }}.tmp &&
+        mv {{ temp_merged_reproducer_var_path }}.tmp
+        {{ temp_merged_reproducer_var_path }}
+      loop: "{{ extra_variable_files }}"
+      loop_control:
+        loop_var: extra_variable_file
+        label: "{{ extra_variable_file | basename }}"
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
     - name: Write back merged reproducer-variables.yml
       ansible.builtin.copy:
-        content: "{{ merged_reproducer_slurp.content | b64decode }}"
+        src: "{{ temp_merged_reproducer_var_path }}"
         dest: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml"
         mode: "0664"
         backup: true
@@ -65,7 +92,7 @@
 
     - name: Overwrite custom-params.yml
       ansible.builtin.copy:
-        content: "{{ merged_reproducer_slurp.content | b64decode }}"
+        src: "{{ temp_merged_reproducer_var_path }}"
         dest: "{{ cifmw_basedir }}/artifacts/parameters/custom-params.yml"
         mode: "0664"
       no_log: "{{ cifmw_nolog | default(true) | bool }}"

roles/reproducer/tasks/premetal.yml

@@ -30,6 +30,15 @@
   loop_control:
     loop_var: secret_file
 
+- name: Pass extra variable files to deployment scripts
+  ansible.builtin.set_fact:
+    cifmw_deploy_architecture_args: >
+      {{ cifmw_deploy_architecture_args | default('') }}
+      -e @{{ extra_variable_file }}
+  loop: "{{ extra_variable_files | default([]) }}"
+  loop_control:
+    loop_var: extra_variable_file
+
 - name: Print final cifmw_deploy_architecture_args
   ansible.builtin.debug:
     msg: >