[Veeam] Check for failures in the restore process (apache#7224)

* Validate failure state in Veeam restore process

* Address Daan review, and properly call method

* Address bryan's reviews

* remove return

Co-authored-by: SadiJr <sadi@scclouds.com.br>
Co-authored-by: João Jandre <48719461+JoaoJandre@users.noreply.github.com>

Author: 3 people

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java

@@ -346,7 +346,7 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
                         String type = pair.second();
                         String path = url.replace(apiURI.toString(), "");
                         if (type.equals("RestoreSession")) {
-                            return checkIfRestoreSessionFinished(type, path);
+                            checkIfRestoreSessionFinished(type, path);
                         }
                     }
                     return true;
@@ -362,17 +362,29 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
         return false;
     }
 
-    protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException {
-        for (int j = 0; j < this.restoreTimeout; j++) {
+
+    /**
+     * Checks the status of the restore session. Checked states are "Success" and "Failure".<br/>
+     * There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,<br/>
+     * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}.
+     */
+    protected void checkIfRestoreSessionFinished(String type, String path) throws IOException {
+        for (int j = 0; j < restoreTimeout; j++) {
             HttpResponse relatedResponse = get(path);
             RestoreSession session = parseRestoreSessionResponse(relatedResponse);
             if (session.getResult().equals("Success")) {
-                return true;
+                return;
             }
+
             if (session.getResult().equalsIgnoreCase("Failed")) {
                 String sessionUid = session.getUid();
+                LOG.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].",
+                        sessionUid, session.getVmDisplayName(),
+                        getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":"))));
                 throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid));
             }
+            LOG.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout));
+
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException ignored) {
@@ -931,6 +943,29 @@ public Pair<Boolean, String> restoreVMToDifferentLocation(String restorePointId,
         return new Pair<>(result.first(), restoreLocation);
     }
 
+    /**
+     * Tries to retrieve the error's description of the Veeam restore task that resulted in an error.
+     * @param uid Session uid in Veeam of the restore process;
+     * @return the description found in Veeam about the cause of error in the restore process.
+     */
+    protected String getRestoreVmErrorDescription(String uid) {
+        LOG.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid));
+        List<String> cmds = Arrays.asList(
+                String.format("$restoreUid = '%s'", uid),
+                "$restore = Get-VBRRestoreSession -Id $restoreUid",
+                "if ($restore) {",
+                    "Write-Output $restore.Description",
+                "} else {",
+                    "Write-Output 'Cannot find restore session with provided uid $restoreUid'",
+                "}"
+        );
+        Pair<Boolean, String> result = executePowerShellCommands(cmds);
+        if (result != null && result.first()) {
+            return result.second();
+        }
+        return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid);
+    }
+
     private boolean isLegacyServer() {
         return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11);
     }

plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java

@@ -59,6 +59,8 @@ public class VeeamClientTest {
     private VeeamClient mockClient;
     private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
 
+    private VeeamClient mock = Mockito.mock(VeeamClient.class);
+
     @Rule
     public WireMockRule wireMockRule = new WireMockRule(9399);
 
@@ -163,7 +165,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti
             Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
             Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession);
             Mockito.when(restoreSession.getResult()).thenReturn("No Success");
-            Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod();
+            Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"));
             mockClient.checkIfRestoreSessionFinished("RestoreTest", "any");
             fail();
         } catch (Exception e) {
@@ -172,6 +174,42 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti
         Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
     }
 
+    @Test
+    public void getRestoreVmErrorDescriptionTestFindErrorDescription() {
+        Pair<Boolean, String> response = new Pair<>(true, "Example of error description found in Veeam.");
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Example of error description found in Veeam.", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() {
+        Pair<Boolean, String> response = new Pair<>(true, "Cannot find restore session with provided uid uuid");
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Cannot find restore session with provided uid uuid", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() {
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
+    }
+
+    @Test
+    public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() {
+        Pair<Boolean, String> response = new Pair<>(false, null);
+        Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+        Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+        String result = mock.getRestoreVmErrorDescription("uuid");
+        Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
+    }
+
+
     private void verifyBackupMetrics(Map<String, Backup.Metric> metrics) {
         Assert.assertEquals(2, metrics.size());