[Genie] 안정성 및 버그 수정 - 생성 대기시간 증가

plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/AutomationControllerManagerImpl.java

@@ -17,6 +17,10 @@
 
 package com.cloud.automation.controller;
 
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
@@ -59,7 +63,6 @@
 import com.cloud.utils.fsm.NoTransitionException;
 import com.cloud.utils.fsm.StateMachine2;
 import com.cloud.utils.net.NetUtils;
-import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.dao.VMInstanceDao;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.SecurityChecker;
@@ -92,6 +95,9 @@
 import static com.cloud.automation.version.AutomationVersionService.AutomationServiceEnabled;
 
 public class AutomationControllerManagerImpl extends ManagerBase implements AutomationControllerService {
+    private static final int AUTOMATION_CONTROLLER_HTTP_PORT = 80;
+    private static final int ALERT_RECOVERY_CONNECT_TIMEOUT_MS = 3000;
+    private static final int ALERT_RECOVERY_READ_TIMEOUT_MS = 3000;
 
     protected StateMachine2<AutomationController.State, AutomationController.Event, AutomationController> _stateMachine = AutomationController.State.getStateMachine();
 
@@ -170,11 +176,8 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat
 
         NetworkVO ntwk = networkDao.findByIdIncludingRemoved(automationController.getNetworkId());
         response.setNetworkId(ntwk.getUuid());
-        response.getAutomationControllerIp(automationController.getAutomationControllerIp());
-        response.getRemoved(automationController.getRemoved());
-        if (automationController.getState() != null) {
-            response.setState(automationController.getState().toString());
-        }
+        response.setAutomationControllerIp(automationController.getAutomationControllerIp());
+        response.setRemoved(automationController.getRemoved());
         DataCenterVO zone = dataCenterDao.findById(automationController.getZoneId());
         if (zone != null) {
             response.setZoneId(zone.getUuid());
@@ -203,7 +206,6 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat
         }
 
         List<UserVmResponse> automationControllerVmResponses = new ArrayList<UserVmResponse>();
-        List<VMInstanceVO> vmList = vmInstanceDao.listByZoneId(automationController.getZoneId());
         List<AutomationControllerVmMapVO> controlVmList = automationControllerVmMapDao.listByAutomationControllerId(automationController.getId());
 
         ResponseObject.ResponseView respView = ResponseObject.ResponseView.Restricted;
@@ -233,33 +235,73 @@ public AutomationControllerResponse addAutomationControllerResponse(long automat
                 }
                 response.setHostName(userVM.getHostName());
             }
-
-            String automationControllerState = String.valueOf(automationController.getState());
-            String automationControllerVmState = automationControllerVmResponses.get(0).getState();
-            try {
-                if (automationControllerState != "Starting") {
-                    if (automationControllerVmState == "Stopped" && automationControllerState != "Stopped") {
-                        stateTransitTo(automationController.getId(), AutomationController.Event.StopRequested);
-                        stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded);
-                    }
-                }
-                if (automationControllerState != "Destroyed" || automationControllerState != "Expunging") {
-                    if (automationControllerVmState == "Destroyed" ) {
-                        stopAutomationController(automationController.getId());
-                    }else if (automationControllerVmState == "Expunging") {
-                        deleteAutomationController(automationController.getId());
-                    }
-                }
-            } catch (Exception e) {
-                logger.warn(String.format("Failed to run Automation controller Alert state scanner on Automation controller : %s status scanner", automationController.getName()), e);
-            }
         }
 
+        response.setState(resolveAutomationControllerResponseState(automationController, automationControllerVmResponses));
         response.setAutomationControllerVms(automationControllerVmResponses);
 
         return response;
     }
 
+    private String resolveAutomationControllerResponseState(AutomationControllerVO automationController,
+                                                            List<UserVmResponse> automationControllerVmResponses) {
+        if (automationController.getState() == null) {
+            return null;
+        }
+        if (!AutomationController.State.Alert.equals(automationController.getState())) {
+            return automationController.getState().toString();
+        }
+        if (automationControllerVmResponses.isEmpty()) {
+            return automationController.getState().toString();
+        }
+
+        final String automationControllerVmState = automationControllerVmResponses.get(0).getState();
+        if (!"Running".equals(automationControllerVmState)) {
+            return automationController.getState().toString();
+        }
+
+        final String address = getAutomationControllerPublicIpAddress(automationController);
+        if (address != null && isAutomationControllerHttpReady(address, AUTOMATION_CONTROLLER_HTTP_PORT)) {
+            return AutomationController.State.Running.toString();
+        }
+
+        return automationController.getState().toString();
+    }
+
+    private String getAutomationControllerPublicIpAddress(AutomationControllerVO automationController) {
+        NetworkVO ntwk = networkDao.findByIdIncludingRemoved(automationController.getNetworkId());
+        if (ntwk == null || ntwk.getGuestType() != Network.GuestType.Isolated) {
+            return null;
+        }
+
+        List<IPAddressVO> ipAddresses = ipAddressDao.listByAssociatedNetwork(ntwk.getId(), true);
+        if (ipAddresses == null || ipAddresses.size() != 1) {
+            return null;
+        }
+
+        return ipAddresses.get(0).getAddress().addr();
+    }
+
+    private boolean isAutomationControllerHttpReady(String address, int port) {
+        HttpURLConnection connection = null;
+        try {
+            URL url = new URL("http://" + address + ":" + port);
+            URLConnection con = url.openConnection();
+            connection = (HttpURLConnection) con;
+            connection.setConnectTimeout(ALERT_RECOVERY_CONNECT_TIMEOUT_MS);
+            connection.setReadTimeout(ALERT_RECOVERY_READ_TIMEOUT_MS);
+            connection.setInstanceFollowRedirects(false);
+            int responseCode = connection.getResponseCode();
+            return responseCode >= 200 && responseCode < 400;
+        } catch (IOException e) {
+            return false;
+        } finally {
+            if (connection != null) {
+                connection.disconnect();
+            }
+        }
+    }
+
     @Override
     public ListResponse<AutomationControllerResponse> listAutomationController(ListAutomationControllerCmd cmd) {
         if (!AutomationServiceEnabled.value()) {

plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStartWorker.java

@@ -33,6 +33,7 @@
 import java.util.List;
 import java.util.Properties;
 import java.net.InetAddress;
+import java.util.concurrent.TimeUnit;
 
 import com.cloud.api.query.vo.UserAccountJoinVO;
 import com.cloud.automation.version.AutomationControllerVersion;
@@ -79,6 +80,12 @@ public class AutomationControllerStartWorker extends AutomationControllerResourc
 
     private AutomationControllerVersion automationControllerVersion;
     private static final long GiB_TO_BYTES = 1024 * 1024 * 1024;
+    private static final int AUTOMATION_CONTROLLER_HTTP_PORT = 80;
+    private static final int HTTP_CONNECT_TIMEOUT_MS = 5000;
+    private static final int HTTP_READ_TIMEOUT_MS = 5000;
+    private static final long AUTOMATION_CONTROLLER_CREATE_READY_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(30);
+    private static final long AUTOMATION_CONTROLLER_READY_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(15);
+    private static final long AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS = TimeUnit.SECONDS.toMillis(5);
 
     public AutomationControllerStartWorker(final AutomationController automationController, final AutomationControllerManagerImpl automationManager) {
         super(automationController, automationManager);
@@ -400,10 +407,9 @@ public boolean startAutomationControllerOnCreate() {
             } catch (Exception e) {
                 throw new RuntimeException(e);
             }
-            boolean urlReachableResult;
             try {
-                urlReachableResult = urlReachable(publicIpAddressStr, 80);
-                if (urlReachableResult == true) {
+                if (waitForAutomationControllerReady(publicIpAddressStr, AUTOMATION_CONTROLLER_HTTP_PORT,
+                        AUTOMATION_CONTROLLER_CREATE_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) {
                     if (logger.isInfoEnabled()) {
                         logger.info(String.format("Starting automation controller : %s", automationController.getName()));
                     }
@@ -432,7 +438,6 @@ public boolean startAutomationControllerOnCreate() {
     public boolean startStoppedAutomationController() throws CloudRuntimeException {
         init();
         IpAddress publicIpAddress = null;
-        boolean urlReachableResult = false;
         publicIpAddress = getAutomationControllerServerIp();
         String publicIpAddressStr = String.valueOf(publicIpAddress.getAddress());
         stateTransitTo(automationController.getId(), AutomationController.Event.StartRequested);
@@ -443,8 +448,8 @@ public boolean startStoppedAutomationController() throws CloudRuntimeException {
             throw new RuntimeException(e);
         }
         try {
-            urlReachableResult = urlReachable(publicIpAddressStr, 80);
-            if (urlReachableResult == true) {
+            if (waitForAutomationControllerReady(publicIpAddressStr, AUTOMATION_CONTROLLER_HTTP_PORT,
+                    AUTOMATION_CONTROLLER_READY_TIMEOUT_MS, AUTOMATION_CONTROLLER_READY_RETRY_INTERVAL_MS)) {
                 if (logger.isInfoEnabled()) {
                     logger.info(String.format("Starting automation controller : %s", automationController.getName()));
                 }
@@ -473,26 +478,51 @@ public boolean pingCheck(String url, int timeout) throws Exception{
         return target.isReachable(timeout);
     }
 
-    public static boolean urlReachable(String address, int port) throws IOException {
-        try {
-            URL url = new URL("http://"+address+":"+port);
-            URLConnection con = url.openConnection();
-            con.setConnectTimeout(400000);
-            con.setReadTimeout(450000);
-            HttpURLConnection exitCode = (HttpURLConnection)con;
-            if(exitCode.getResponseCode() == 200) {
+    private boolean waitForAutomationControllerReady(String address, int port, long timeoutMs, long retryIntervalMs) {
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        while (System.currentTimeMillis() < deadline) {
+            int responseCode = getHttpResponseCode(address, port);
+            if (responseCode >= 200 && responseCode < 400) {
                 return true;
             }
-            else {
+
+            if (logger.isDebugEnabled()) {
+                logger.debug(String.format("Automation controller %s is not ready yet on %s:%d (HTTP %d)",
+                        automationController.getName(), address, port, responseCode));
+            }
+
+            try {
+                Thread.sleep(retryIntervalMs);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                logger.warn(String.format("Interrupted while waiting for automation controller %s readiness", automationController.getName()), e);
                 return false;
             }
-        } catch (java.net.SocketTimeoutException e) {
-            return false;
-        } catch (IOException exception) {
-            return false;
         }
+
+        logger.warn(String.format("Timed out waiting for automation controller %s readiness on %s:%d after %d ms",
+                automationController.getName(), address, port, timeoutMs));
+        return false;
     }
 
+    private static int getHttpResponseCode(String address, int port) {
+        HttpURLConnection connection = null;
+        try {
+            URL url = new URL("http://" + address + ":" + port);
+            URLConnection con = url.openConnection();
+            connection = (HttpURLConnection) con;
+            connection.setConnectTimeout(HTTP_CONNECT_TIMEOUT_MS);
+            connection.setReadTimeout(HTTP_READ_TIMEOUT_MS);
+            connection.setInstanceFollowRedirects(false);
+            return connection.getResponseCode();
+        } catch (IOException exception) {
+            return -1;
+        } finally {
+            if (connection != null) {
+                connection.disconnect();
+            }
+        }
+    }
 
 
 }

plugins/integrations/automation-service/src/main/java/com/cloud/automation/controller/actionworkers/AutomationControllerStopWorker.java

@@ -30,8 +30,12 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.TimeUnit;
 
 public class AutomationControllerStopWorker extends AutomationControllerActionWorker {
+    private static final long AUTOMATION_CONTROLLER_STOP_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(10);
+    private static final long AUTOMATION_CONTROLLER_STOP_RETRY_INTERVAL_MS = TimeUnit.SECONDS.toMillis(5);
+
     public AutomationControllerStopWorker(final AutomationController automationController, final AutomationControllerManagerImpl automationManager) {
         super(automationController, automationManager);
     }
@@ -60,13 +64,42 @@ public boolean stop() throws CloudRuntimeException {
             }
         }
         for (final UserVm userVm : automationControllerVMs) {
-            UserVm vm = userVmDao.findById(userVm.getId());
-            if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) {
+            if (!waitForVmStopped(userVm.getId(), AUTOMATION_CONTROLLER_STOP_TIMEOUT_MS,
+                    AUTOMATION_CONTROLLER_STOP_RETRY_INTERVAL_MS)) {
                 logTransitStateAndThrow(Level.ERROR, String.format("Failed to stop VMs in automation controller : %s",
                 automationController.getName()), automationController.getId(), AutomationController.Event.OperationFailed);
             }
         }
         stateTransitTo(automationController.getId(), AutomationController.Event.OperationSucceeded);
         return true;
     }
+
+    private boolean waitForVmStopped(long vmId, long timeoutMs, long retryIntervalMs) {
+        long deadline = System.currentTimeMillis() + timeoutMs;
+        while (System.currentTimeMillis() < deadline) {
+            UserVm vm = userVmDao.findById(vmId);
+            if (vm != null && vm.getState().equals(VirtualMachine.State.Stopped)) {
+                return true;
+            }
+
+            if (logger.isDebugEnabled()) {
+                String currentState = vm == null ? "null" : vm.getState().toString();
+                logger.debug(String.format("VM %d for automation controller %s is not stopped yet (state=%s)",
+                        vmId, automationController.getName(), currentState));
+            }
+
+            try {
+                Thread.sleep(retryIntervalMs);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                logger.warn(String.format("Interrupted while waiting for automation controller %s VM %d to stop",
+                        automationController.getName(), vmId), e);
+                return false;
+            }
+        }
+
+        logger.warn(String.format("Timed out waiting for automation controller %s VM %d to stop after %d ms",
+                automationController.getName(), vmId, timeoutMs));
+        return false;
+    }
 }

plugins/integrations/automation-service/src/main/java/org/apache/cloudstack/api/response/AutomationControllerResponse.java

@@ -268,7 +268,9 @@ public String getAutomationControllerIp(String automationControllerIp) {
     public void setAutomationControllerIp(String automationControllerIp) {
         this.automationControllerIp = automationControllerIp;
     }
-    public void getRemoved(Date removed) {
+
+    public void setRemoved(Date removed) {
+        this.removed = removed;
     }
 
     public String getIpAddress(String ipAddress) {

plugins/integrations/automation-service/src/main/resources/conf/genie.yml

@@ -26,7 +26,13 @@ write_files:
     {{ genie_encode }}
 
 runcmd:
-  - wget https://raw.githubusercontent.com/ablecloud-team/ablestack-genie/master/genie-shell/automation_controller_template/deploy_automation_controller.yml -P /root/
-  - ansible-playbook /root/deploy_automation_controller.yml
-#  - cp -r /genie/deploy_automation_controller.yml /root/
-#  - ansible-playbook /root/deploy_automation_controller.yml
+  - |
+      echo "[genie] downloading deploy_automation_controller.yml to /root" | tee -a /var/log/genie-bootstrap.log
+      wget --tries=10 --waitretry=5 https://raw.githubusercontent.com/ablecloud-team/ablestack-genie/master/genie-shell/automation_controller_template/deploy_automation_controller.yml -O /root/deploy_automation_controller.yml
+      status=$?
+      echo "[genie] wget exit code: ${status}" | tee -a /var/log/genie-bootstrap.log
+      ls -l /root/deploy_automation_controller.yml | tee -a /var/log/genie-bootstrap.log
+      exit ${status}
+  - |
+      echo "[genie] running ansible-playbook /root/deploy_automation_controller.yml" | tee -a /var/log/genie-bootstrap.log
+      ansible-playbook /root/deploy_automation_controller.yml 2>&1 | tee -a /var/log/genie-bootstrap.log