Indirect agent connection improvements, includes the following improvements.

- Enhances the Host connecting logic to avoid connecting storm (where Agent opens multiple sockets against Management Server).
- Implements HostConnectProcess task where Host upon connection checks whether lock is available, traces Host connecting progress, status and timeout.
- Introduces AgentConnectStatusCommand, where Host checks whether lock for the Host is available (i.e. "previous" connect process is finished).
- Implementes logic to check whether Management Server has lock against Host (exposed MySQL DB lock presence via API)
- Removes synchronization on Host disconnect process, double-disconnect logic in clustered Management Server environment, added early removal from ping map (in case of combination ping timeout delay + synchronized disconnect process the Agent Manager submits more disconnect requests)
- Introduces parameterized connection and status check timeouts
- Implements backoff algorithm abstraction - can be used either constant backoff timeout or exponential with jitter to wait between connection Host attempts to Management Server
- Implements ServerAttache to be used on the Agent side of communication (similar to Attache on Management Server side)
- Enhances/Adds logs significantly to Host Agent and Agent Manager logic to trace Host connecting and disconnecting process, including ids, names, context UUIDs and timings (how much time took overall initialization/deinitialization)
- Adds logs to communication between Management Servers (PDU requests)
- Adds DB indexes to improve search performance, uses IDEMPOTENT_ADD_INDEX for safer DB schema updates

Author: mprokopchuk

agent/src/main/java/com/cloud/agent/Agent.java

@@ -32,6 +32,7 @@
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.utils.backoff.BackoffFactory;
 import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.daemon.DaemonInitException;
@@ -52,7 +53,6 @@
 import com.cloud.utils.ProcessUtil;
 import com.cloud.utils.PropertiesUtil;
 import com.cloud.utils.backoff.BackoffAlgorithm;
-import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public class AgentShell implements IAgentShell, Daemon {
@@ -95,6 +95,16 @@ public BackoffAlgorithm getBackoffAlgorithm() {
         return _backoff;
     }
 
+    @Override
+    public void setBackoffAlgorithm(BackoffAlgorithm backoffAlgorithm) {
+        this._backoff = backoffAlgorithm;
+        try {
+            backoffAlgorithm.getConfiguration().forEach((key, value) -> setPersistentProperty(null, key, value));
+        } catch (RuntimeException e) {
+            LOGGER.warn("Failed to persist backoff properties");
+        }
+    }
+
     @Override
     public int getPingRetries() {
         return _pingRetries;
@@ -400,7 +410,7 @@ public void init(String[] args) throws ConfigurationException {
         final Class<?> c = this.getClass();
         _version = c.getPackage().getImplementationVersion();
         if (_version == null) {
-            throw new CloudRuntimeException("Unable to find the implementation version of this agent");
+            _version = "unknown";
         }
         LOGGER.info("Implementation Version is {}", _version);
 
@@ -410,7 +420,7 @@ public void init(String[] args) throws ConfigurationException {
         if (LOGGER.isDebugEnabled()) {
             List<String> properties = Collections.list((Enumeration<String>)_properties.propertyNames());
             for (String property : properties) {
-                LOGGER.debug("Found property: {}", property);
+                LOGGER.debug("Found property: {}, value: {}", property, _properties.getProperty(property));
             }
         }
 
@@ -424,11 +434,15 @@ public void init(String[] args) throws ConfigurationException {
             _properties.put(cmdLineProp.getKey(), cmdLineProp.getValue());
         }
 
-        LOGGER.info("Defaulting to the constant time backoff algorithm");
-        _backoff = new ConstantTimeBackoff();
-        Map<String, Object> map = new HashMap<>();
-        map.put("seconds", _properties.getProperty("backoff.seconds"));
-        _backoff.configure("ConstantTimeBackoff", map);
+        try {
+            LOGGER.info("Creating backoff delay algorithm implementation");
+            setBackoffAlgorithm(BackoffFactory.create(_properties));
+            LOGGER.info("Created {} delay algorithm implementation", getBackoffAlgorithm().getClass().getName());
+        } catch (RuntimeException e) {
+            String msg = String.format("Failed to create backoff with provided properties %s, failing back to default", _properties);
+            LOGGER.warn(msg, e);
+            setBackoffAlgorithm(BackoffFactory.createDefault(_properties));
+        }
     }
 
     private void launchAgent() throws ConfigurationException {
@@ -546,7 +560,7 @@ public void start() {
             }
 
         } catch (final Exception e) {
-            LOGGER.error("Unable to start agent: ", e);
+            LOGGER.error("Unable to start agent: {}", e.getLocalizedMessage(), e);
             System.exit(ExitStatus.Error.value());
         }
     }
@@ -568,6 +582,7 @@ public static void main(String[] args) {
             shell.init(args);
             shell.start();
         } catch (ConfigurationException e) {
+            LOGGER.fatal(e.getMessage(), e);
             System.out.println(e.getMessage());
         }
     }