Skip to content

Commit 7dd756b

Browse files
stuggiclaude
stuggi
and
claude
committed
[b/r] Add full backup playbook and update README
Add backup-openstack.yaml playbook that orchestrates a complete backup: 1. Trigger Galera DB dumps via cronjobs 2. OADP PVC backup (CSI snapshots of auto-labeled PVCs) 3. OADP resources backup (all resources except PVCs) PVC labeling is handled automatically by service operators (glance-operator, mariadb-operator) - no manual labeling needed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f2c01e2 commit 7dd756b

2 files changed

Lines changed: 257 additions & 0 deletions

File tree

‎docs/dev/webhook/backup/README.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,33 @@ We use a two-backup strategy:
3131

3232
## Quick Start
3333

34+
### Automated (Recommended)
35+
36+
Use the backup playbook to orchestrate the full backup flow:
37+
38+
```bash
39+
ansible-playbook docs/dev/webhook/backup/backup-openstack.yaml
40+
```
41+
42+
The playbook runs three steps:
43+
1. **Trigger Galera DB dumps** — creates jobs from GaleraBackup cronjobs
44+
2. **OADP PVC backup** — CSI snapshots of PVCs labeled with `openstack.org/backup=true`
45+
3. **OADP resources backup** — all resources in the namespace except PVCs
46+
47+
PVC backup labels are set automatically by service operators (glance-operator, mariadb-operator).
48+
49+
Override defaults with extra vars:
50+
```bash
51+
ansible-playbook docs/dev/webhook/backup/backup-openstack.yaml \
52+
-e openstack_namespace=openstack \
53+
-e storage_location=velero-1 \
54+
-e backup_ttl=168h
55+
```
56+
57+
### Manual
58+
59+
Apply the backup CRs directly:
60+
3461
```bash
3562
# Create both backups
3663
oc apply -f backup-openstack-resources.yaml
@@ -44,6 +71,8 @@ oc describe backup openstack-backup-resources -n openshift-adp
4471
oc describe backup openstack-backup-pvcs -n openshift-adp
4572
```
4673

74+
**Note:** When running manually, trigger Galera DB dumps first to ensure fresh database backups on the PVCs before the CSI snapshot.
75+
4776
## Backup Contents
4877

4978
### backup-openstack-pvcs

‎docs/dev/webhook/backup/backup-openstack.yaml‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
---
2+
# Full OpenStack Backup Playbook
3+
#
4+
# Orchestrates a complete backup of an OpenStack control plane:
5+
# 1. Trigger Galera database dumps (creates fresh DB dumps on PVCs)
6+
# 2. OADP PVC backup (CSI snapshots of labeled PVCs)
7+
# 3. OADP resources backup (CRs, Secrets, ConfigMaps, NADs, etc.)
8+
#
9+
# PVC backup labels (openstack.org/backup=true) are set automatically
10+
# by the service operators (glance-operator, mariadb-operator, etc.).
11+
#
12+
# Prerequisites:
13+
# - OADP operator installed and configured (see docs/dev/setup-oadp-minio.md)
14+
# - GaleraBackup CRs created (cronjobs exist for DB dumps)
15+
# - VolumeSnapshotClass with velero label configured
16+
# - PVC sizes use binary units (Gi, not G) - see OSPRH-27441
17+
#
18+
# Usage:
19+
# ansible-playbook docs/dev/webhook/backup/backup-openstack.yaml
20+
#
21+
# Variables:
22+
# openstack_namespace: Target namespace (default: openstack)
23+
# oadp_namespace: OADP namespace (default: openshift-adp)
24+
# backup_name_suffix: Suffix for backup names (default: timestamp)
25+
# galera_backup_timeout: Timeout for DB dump jobs (default: 10m)
26+
# oadp_backup_timeout: Timeout for OADP backups (default: 30m)
27+
# storage_location: Velero storage location (default: velero-1)
28+
# backup_ttl: Backup retention (default: 720h / 30 days)
29+
30+
- name: OpenStack Full Backup
31+
hosts: localhost
32+
gather_facts: false
33+
34+
vars:
35+
openstack_namespace: "{{ openstack_namespace | default('openstack') }}"
36+
oadp_namespace: "{{ oadp_namespace | default('openshift-adp') }}"
37+
backup_name_suffix: "{{ backup_name_suffix | default(lookup('pipe', 'date +%Y%m%d-%H%M%S')) }}"
38+
galera_backup_timeout: "{{ galera_backup_timeout | default('10m') }}"
39+
oadp_backup_timeout: "{{ oadp_backup_timeout | default('30m') }}"
40+
storage_location: "{{ storage_location | default('velero-1') }}"
41+
backup_ttl: "{{ backup_ttl | default('720h') }}"
42+
43+
tasks:
44+
# ========================================
45+
# Step 1: Trigger Galera Database Dumps
46+
# ========================================
47+
- name: "Step 1: Trigger Galera database dumps"
48+
ansible.builtin.debug:
49+
msg:
50+
- "========================================"
51+
- "Step 1: Trigger Galera Database Dumps"
52+
- "========================================"
53+
- "Creating jobs from GaleraBackup cronjobs to dump databases to PVCs"
54+
55+
- name: Get list of GaleraBackup cronjobs
56+
ansible.builtin.shell: |
57+
oc get cronjob -n {{ openstack_namespace }} -l app=galera -o jsonpath='{.items[*].metadata.name}'
58+
register: galera_backup_cronjobs
59+
changed_when: false
60+
61+
- name: Print GaleraBackup cronjobs found
62+
ansible.builtin.debug:
63+
msg: "Found GaleraBackup cronjobs: {{ galera_backup_cronjobs.stdout.split() if galera_backup_cronjobs.stdout != '' else 'none' }}"
64+
65+
- name: Trigger Galera backup jobs
66+
ansible.builtin.shell: |
67+
BACKUP_JOB_NAME="{{ item }}-{{ backup_name_suffix }}"
68+
oc -n {{ openstack_namespace }} create job --from=cronjob/{{ item }} ${BACKUP_JOB_NAME}
69+
echo ${BACKUP_JOB_NAME}
70+
loop: "{{ galera_backup_cronjobs.stdout.split() }}"
71+
register: galera_backup_jobs
72+
changed_when: true
73+
when: galera_backup_cronjobs.stdout != ""
74+
75+
- name: Wait for Galera backup jobs to complete
76+
ansible.builtin.shell: |
77+
oc -n {{ openstack_namespace }} wait --for=condition=complete job/{{ item.stdout_lines[-1] }} --timeout={{ galera_backup_timeout }}
78+
loop: "{{ galera_backup_jobs.results }}"
79+
changed_when: false
80+
when: galera_backup_cronjobs.stdout != ""
81+
82+
- name: Print Step 1 result
83+
ansible.builtin.debug:
84+
msg: "Galera database backups completed"
85+
when: galera_backup_cronjobs.stdout != ""
86+
87+
- name: Print Step 1 skip
88+
ansible.builtin.debug:
89+
msg: "No GaleraBackup cronjobs found - skipping database dump"
90+
when: galera_backup_cronjobs.stdout == ""
91+
92+
# ========================================
93+
# Step 2: OADP PVC Backup (CSI Snapshots)
94+
# ========================================
95+
- name: "Step 2: OADP PVC backup"
96+
ansible.builtin.debug:
97+
msg:
98+
- "========================================"
99+
- "Step 2: OADP PVC Backup (CSI Snapshots)"
100+
- "========================================"
101+
- "PVCs are auto-labeled by operators (glance-operator, mariadb-operator)"
102+
103+
- name: List PVCs marked for backup
104+
ansible.builtin.shell: |
105+
oc get pvc -n {{ openstack_namespace }} -l openstack.org/backup=true \
106+
-o custom-columns=NAME:.metadata.name,SIZE:.spec.resources.requests.storage,SERVICE:.metadata.labels.service --no-headers
107+
register: labeled_pvcs
108+
changed_when: false
109+
failed_when: false
110+
111+
- name: Print labeled PVCs
112+
ansible.builtin.debug:
113+
msg: "{{ labeled_pvcs.stdout_lines }}"
114+
when: labeled_pvcs.stdout_lines | default([]) | length > 0
115+
116+
- name: Warn if no PVCs labeled
117+
ansible.builtin.debug:
118+
msg: "WARNING: No PVCs found with openstack.org/backup=true label"
119+
when: labeled_pvcs.stdout_lines | default([]) | length == 0
120+
121+
- name: Create OADP PVC backup
122+
ansible.builtin.shell: |
123+
cat <<EOF | oc apply -f -
124+
apiVersion: velero.io/v1
125+
kind: Backup
126+
metadata:
127+
name: openstack-backup-pvcs-{{ backup_name_suffix }}
128+
namespace: {{ oadp_namespace }}
129+
spec:
130+
includedNamespaces:
131+
- {{ openstack_namespace }}
132+
labelSelector:
133+
matchLabels:
134+
openstack.org/backup: "true"
135+
snapshotVolumes: true
136+
defaultVolumesToFsBackup: false
137+
volumeSnapshotLocations: []
138+
storageLocation: {{ storage_location }}
139+
ttl: {{ backup_ttl }}
140+
EOF
141+
changed_when: true
142+
143+
- name: Wait for PVC backup to complete
144+
ansible.builtin.shell: |
145+
oc wait --for=jsonpath='{.status.phase}'=Completed \
146+
backup/openstack-backup-pvcs-{{ backup_name_suffix }} \
147+
-n {{ oadp_namespace }} \
148+
--timeout={{ oadp_backup_timeout }}
149+
changed_when: false
150+
151+
- name: Get PVC backup status
152+
ansible.builtin.shell: |
153+
oc get backup openstack-backup-pvcs-{{ backup_name_suffix }} -n {{ oadp_namespace }} \
154+
-o jsonpath='{.status.phase}'
155+
register: pvc_backup_status
156+
changed_when: false
157+
158+
- name: Print Step 2 result
159+
ansible.builtin.debug:
160+
msg: "PVC backup status: {{ pvc_backup_status.stdout }}"
161+
162+
# ========================================
163+
# Step 3: OADP Resources Backup
164+
# ========================================
165+
- name: "Step 3: OADP resources backup"
166+
ansible.builtin.debug:
167+
msg:
168+
- "========================================"
169+
- "Step 3: OADP Resources Backup"
170+
- "========================================"
171+
- "Backing up all resources except PVCs"
172+
173+
- name: Create OADP resources backup
174+
ansible.builtin.shell: |
175+
cat <<EOF | oc apply -f -
176+
apiVersion: velero.io/v1
177+
kind: Backup
178+
metadata:
179+
name: openstack-backup-resources-{{ backup_name_suffix }}
180+
namespace: {{ oadp_namespace }}
181+
spec:
182+
includedNamespaces:
183+
- {{ openstack_namespace }}
184+
excludedResources:
185+
- persistentvolumeclaims
186+
- persistentvolumes
187+
storageLocation: {{ storage_location }}
188+
ttl: {{ backup_ttl }}
189+
EOF
190+
changed_when: true
191+
192+
- name: Wait for resources backup to complete
193+
ansible.builtin.shell: |
194+
oc wait --for=jsonpath='{.status.phase}'=Completed \
195+
backup/openstack-backup-resources-{{ backup_name_suffix }} \
196+
-n {{ oadp_namespace }} \
197+
--timeout={{ oadp_backup_timeout }}
198+
changed_when: false
199+
200+
- name: Get resources backup status
201+
ansible.builtin.shell: |
202+
oc get backup openstack-backup-resources-{{ backup_name_suffix }} -n {{ oadp_namespace }} \
203+
-o jsonpath='{.status.phase}'
204+
register: resources_backup_status
205+
changed_when: false
206+
207+
- name: Print Step 3 result
208+
ansible.builtin.debug:
209+
msg: "Resources backup status: {{ resources_backup_status.stdout }}"
210+
211+
# ========================================
212+
# Summary
213+
# ========================================
214+
- name: Print backup summary
215+
ansible.builtin.debug:
216+
msg:
217+
- "========================================"
218+
- "Backup Complete"
219+
- "========================================"
220+
- ""
221+
- "Backup name suffix: {{ backup_name_suffix }}"
222+
- "PVC backup: openstack-backup-pvcs-{{ backup_name_suffix }}"
223+
- "Resources backup: openstack-backup-resources-{{ backup_name_suffix }}"
224+
- ""
225+
- "To verify:"
226+
- " oc get backup -n {{ oadp_namespace }} | grep {{ backup_name_suffix }}"
227+
- ""
228+
- "To restore, see: docs/dev/webhook/restore/README.md"

0 commit comments

Comments
 (0)