server: Prevent vm snapshots being indefinitely stuck in Expunging state on deletion failure (#4898)

Fixes #4201

This PR addresses the issue of a vm snapshot being indefinitely stuck is Expunging state in case deletion fails. 

Co-authored-by: Pearl Dsilva <pearl.dsilva@shapeblue.com>

Author: Pearl1594

api/src/main/java/com/cloud/vm/snapshot/VMSnapshot.java

@@ -59,6 +59,7 @@ public String getDescription() {
             s_fsm.addTransition(Error, Event.ExpungeRequested, Expunging);
             s_fsm.addTransition(Expunging, Event.ExpungeRequested, Expunging);
             s_fsm.addTransition(Expunging, Event.OperationSucceeded, Removed);
+            s_fsm.addTransition(Expunging, Event.OperationFailed, Error);
         }
     }
 

engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java

@@ -230,11 +230,10 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) {
             } else {
                 String errMsg = (answer == null) ? null : answer.getDetails();
                 s_logger.error("Delete vm snapshot " + vmSnapshot.getName() + " of vm " + userVm.getInstanceName() + " failed due to " + errMsg);
+                processAnswer(vmSnapshotVO, userVm, answer, hostId);
                 throw new CloudRuntimeException("Delete vm snapshot " + vmSnapshot.getName() + " of vm " + userVm.getInstanceName() + " failed due to " + errMsg);
             }
-        } catch (OperationTimedoutException e) {
-            throw new CloudRuntimeException("Delete vm snapshot " + vmSnapshot.getName() + " of vm " + userVm.getInstanceName() + " failed due to " + e.getMessage());
-        } catch (AgentUnavailableException e) {
+        } catch (OperationTimedoutException | AgentUnavailableException e) {
             throw new CloudRuntimeException("Delete vm snapshot " + vmSnapshot.getName() + " of vm " + userVm.getInstanceName() + " failed due to " + e.getMessage());
         }
     }
@@ -254,9 +253,13 @@ public void doInTransactionWithoutResult(TransactionStatus status) throws NoTran
                         finalizeRevert(vmSnapshot, answer.getVolumeTOs());
                         vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationSucceeded);
                     } else if (as instanceof DeleteVMSnapshotAnswer) {
-                        DeleteVMSnapshotAnswer answer = (DeleteVMSnapshotAnswer)as;
-                        finalizeDelete(vmSnapshot, answer.getVolumeTOs());
-                        vmSnapshotDao.remove(vmSnapshot.getId());
+                        if (as.getResult()) {
+                            DeleteVMSnapshotAnswer answer = (DeleteVMSnapshotAnswer) as;
+                            finalizeDelete(vmSnapshot, answer.getVolumeTOs());
+                            vmSnapshotDao.remove(vmSnapshot.getId());
+                        } else {
+                            vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationFailed);
+                        }
                     }
                 }
             });