B #7918: Clean up Ceph backup snapshots on failure (#4321)

Two fixes for temporary RBD snapshots left over by failed backups:

- Remove a stale snapshot with the same name before 'rbd snap create'
- Add an EXIT trap to the generated backup script that removes the
  snapshots created by the failed run.

Author: xorel

src/tm_mad/lib/backup.rb

@@ -49,8 +49,9 @@ def backup_disks_sh(options = {})
             snap_cmd = ''
             expo_cmd = ''
 
-            snap_clup = ''
-            expo_clup = ''
+            snap_clup  = ''
+            expo_clup  = ''
+            snap_abort = ''
             onebex_cmd = ''
 
             xml_vm = REXML::Document.new(@xml).root
@@ -79,10 +80,11 @@ def backup_disks_sh(options = {})
                 cmds = d.backup_cmds(backup_dir, ds, live, disk_format)
                 return nil unless cmds
 
-                snap_cmd  << cmds[:snapshot].to_s
-                expo_cmd  << cmds[:export].to_s
-                snap_clup << cmds[:snapshot_clup].to_s
-                expo_clup << cmds[:export_clup].to_s
+                snap_cmd   << cmds[:snapshot].to_s
+                expo_cmd   << cmds[:export].to_s
+                snap_clup  << cmds[:snapshot_clup].to_s
+                expo_clup  << cmds[:export_clup].to_s
+                snap_abort << cmds[:snapshot_abort].to_s
 
                 next unless cmds[:start_onebex]
 
@@ -103,6 +105,23 @@ def backup_disks_sh(options = {})
             <<~EOS
                 set -ex -o pipefail
 
+                # ---------------------------------------------------------
+                # Remove disk snapshots created by this backup on failure
+                # ---------------------------------------------------------
+                abort_cleanup() {
+                    rc=$?
+
+                    [ "$rc" = "0" ] && exit 0
+
+                    set +e
+
+                    #{snap_abort}
+
+                    exit "$rc"
+                }
+
+                trap abort_cleanup EXIT
+
                 # ----------------------
                 # Prepare backup folder
                 # ----------------------

src/tm_mad/lib/ceph.rb

@@ -141,8 +141,9 @@ def backup_cmds(backup_dir, ds, live, format = nil)
                 snap_cmd = ''
                 expo_cmd = ''
 
-                snap_clup = ''
-                expo_clup = ''
+                snap_clup  = ''
+                expo_clup  = ''
+                snap_abort = ''
 
                 backup_util = '/var/tmp/one/tm/lib/backup_rbd.rb'
 
@@ -154,12 +155,16 @@ def backup_cmds(backup_dir, ds, live, format = nil)
                     if live
                         snapshot = "#{@rbd_image}@one_backup"
 
+                        # Remove stale snapshot left over by a failed backup
+                        snap_cmd << rm_snaps_sh({ :type => :eq, :text => 'one_backup' })
                         snap_cmd << "#{@rbd_cmd} snap create #{snapshot}\n"
                         expo_cmd << ds.cmd_confinement(
                             "#{@rbd_cmd} export #{snapshot} #{draw}\n",
                             backup_dir
                         )
                         snap_clup << "#{@rbd_cmd} snap rm #{snapshot}\n"
+
+                        snap_abort << rm_snaps_sh({ :type => :eq, :text => 'one_backup' })
                     else
                         expo_cmd << ds.cmd_confinement(
                             "#{@rbd_cmd} export #{@rbd_image} #{draw}\n",
@@ -195,6 +200,8 @@ def backup_cmds(backup_dir, ds, live, format = nil)
                         "ruby #{backup_util} #{@rbd_image} NONE #{snapshot} #{dexp}\n",
                         backup_dir
                     )
+
+                    snap_abort << rm_snaps_sh({ :type => :prefix, :text => INC_SNAP_PREFIX })
                 else
                     # Incremental backup
                     incid = @vm_backup_config[:last_increment] + 1
@@ -204,6 +211,8 @@ def backup_cmds(backup_dir, ds, live, format = nil)
 
                     last_snap = "one_backup_#{@vm_backup_config[:last_increment]}"
 
+                    # Remove stale snapshot left over by a failed backup
+                    snap_cmd << rm_snaps_sh({ :type => :eq, :text => "one_backup_#{incid}" })
                     snap_cmd << "#{@rbd_cmd} snap create #{snapshot}\n"
                     snap_cmd << "#{@rbd_cmd} snap protect #{snapshot}\n"
 
@@ -225,14 +234,19 @@ def backup_cmds(backup_dir, ds, live, format = nil)
 
                     old_snapshot = "one_backup_#{@vm_backup_config[:last_increment]}"
                     snap_clup << rm_snaps_sh({ :type => :eq, :text => old_snapshot })
+
+                    # On abort remove only the new snapshot, the previous one
+                    # is still the base for the next increment
+                    snap_abort << rm_snaps_sh({ :type => :eq, :text => "one_backup_#{incid}" })
                 end
 
                 {
-                    :snapshot      => snap_cmd,
-                    :export        => expo_cmd,
-                    :snapshot_clup => snap_clup,
-                    :export_clup   => expo_clup,
-                    :cleanup       => snap_clup + expo_clup
+                    :snapshot       => snap_cmd,
+                    :export         => expo_cmd,
+                    :snapshot_clup  => snap_clup,
+                    :export_clup    => expo_clup,
+                    :snapshot_abort => snap_abort,
+                    :cleanup        => snap_clup + expo_clup
                 }
             end