diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
index 89e080eb4d..db4788c08d 100644
--- a/etc/kayobe/kolla.yml
+++ b/etc/kayobe/kolla.yml
@@ -844,7 +844,11 @@ kolla_enable_prometheus: true
 
 # Extra items to add to kolla_openstack_custom_config_rules_default
 # to produce kolla_openstack_custom_config_rules.
-#kolla_openstack_custom_config_rules_extra:
+kolla_openstack_custom_config_rules_extra:
+  # Files ending with .example should not be copied
+  - glob: "**/*.example"
+    strategy: noop
+    priority: 1
 
 ###############################################################################
 # Passwords and credentials.
diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example
index f41401e886..dcce25ccf3 100644
--- a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example
+++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example
@@ -18,11 +18,11 @@ route:
 receivers:
   - name: 'msteamsv2-notifications'
     msteamsv2_configs:
-    - webhook_url: '{{ secrets_msteams_notification_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}'
+    - webhook_url: '{{ secrets_msteams_notification_channel_url | ansible.builtin.mandatory(msg="secrets_msteams_notification_channel_url is undefined. Please add this variable to secrets.yml when using the Microsoft Teams Alertmanager integration.") }}'
       send_resolved: true
   - name: 'msteamsv2-critical-notifications'
     msteamsv2_configs:
-    - webhook_url: '{{ secrets_msteams_notification_critical_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}'
+    - webhook_url: '{{ secrets_msteams_notification_critical_channel_url | ansible.builtin.mandatory(msg="secrets_msteams_notification_critical_channel_url is undefined. Please add this variable to secrets.yml when using the Microsoft Teams Alertmanager integration.") }}'
       send_resolved: true
 
 templates: