framework/db: use HikariCP as default and improvements (apache#9518)

Per docs, if the mysql connector is JDBC2 compliant then it should use
the Connection.isValid API to test a connection.
(https://docs.oracle.com/javase/8/docs/api/java/sql/Connection.html#isValid-int-)

This would significantly reduce query lags and API throughput, as for
every SQL query one or two SELECT 1 are performed everytime a Connection
is given to application logic.

This should only be accepted when the driver is JDBC4 complaint.

As per the docs, the connector-j can use /* ping */ before calling
SELECT 1 to have light weight application pings to the server:
https://dev.mysql.com/doc/connector-j/en/connector-j-usagenotes-j2ee-concepts-connection-pooling.html

Replaces dbcp2 connection pool library with more performant HikariCP.
With this unit tests are failing but build is passing.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>
Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Rohit Yadav <rohityadav89@gmail.com>

Author: shwstppr

client/conf/db.properties.in

@@ -34,10 +34,14 @@ db.cloud.uri=
 
 
 # CloudStack database tuning parameters
+db.cloud.connectionPoolLib=hikaricp
 db.cloud.maxActive=250
 db.cloud.maxIdle=30
-db.cloud.maxWait=10000
-db.cloud.validationQuery=SELECT 1
+db.cloud.maxWait=600000
+db.cloud.minIdleConnections=5
+db.cloud.connectionTimeout=30000
+db.cloud.keepAliveTime=600000
+db.cloud.validationQuery=/* ping */ SELECT 1
 db.cloud.testOnBorrow=true
 db.cloud.testWhileIdle=true
 db.cloud.timeBetweenEvictionRunsMillis=40000
@@ -70,9 +74,13 @@ db.usage.uri=
 
 
 # usage database tuning parameters
+db.usage.connectionPoolLib=hikaricp
 db.usage.maxActive=100
 db.usage.maxIdle=30
-db.usage.maxWait=10000
+db.usage.maxWait=600000
+db.usage.minIdleConnections=5
+db.usage.connectionTimeout=30000
+db.usage.keepAliveTime=600000
 db.usage.url.params=serverTimezone=UTC
 
 # Simulator database settings
@@ -82,9 +90,13 @@ db.simulator.host=@DBHOST@
 db.simulator.driver=@DBDRIVER@
 db.simulator.port=3306
 db.simulator.name=simulator
+db.simulator.connectionPoolLib=hikaricp
 db.simulator.maxActive=250
 db.simulator.maxIdle=30
-db.simulator.maxWait=10000
+db.simulator.maxWait=600000
+db.simulator.minIdleConnections=5
+db.simulator.connectionTimeout=30000
+db.simulator.keepAliveTime=600000
 db.simulator.autoReconnect=true
 
 # Connection URI to the database "simulator". When this property is set, only the following properties will be used along with it: db.simulator.host, db.simulator.port, db.simulator.name, db.simulator.autoReconnect. Other properties will be ignored.

developer/pom.xml

@@ -42,6 +42,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-pool2</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.jasypt</groupId>
             <artifactId>jasypt</artifactId>

engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java

@@ -116,10 +116,6 @@ private static void initDB(String dbPropsFile, String rootPassword, String[] dat
     }
 
     public static void main(String[] args) {
-
-        ClassPathXmlApplicationContext appContext = new ClassPathXmlApplicationContext(new String[] {"/com/cloud/upgrade/databaseCreatorContext.xml"});
-        appContext.getBean(ComponentContext.class);
-
         String dbPropsFile = "";
         List<String> sqlFiles = new ArrayList<String>();
         List<String> upgradeClasses = new ArrayList<String>();
@@ -166,13 +162,17 @@ public static void main(String[] args) {
             System.exit(1);
         }
 
+        initDB(dbPropsFile, rootPassword, databases, dryRun);
+
+        ClassPathXmlApplicationContext appContext = new ClassPathXmlApplicationContext(new String[] {"/com/cloud/upgrade/databaseCreatorContext.xml"});
+        appContext.getBean(ComponentContext.class);
+
         try {
             TransactionLegacy.initDataSource(dbPropsFile);
         } catch (IOException e) {
             e.printStackTrace();
             System.exit(1);
         }
-        initDB(dbPropsFile, rootPassword, databases, dryRun);
 
         // Process sql files
         for (String sqlFile : sqlFiles) {

engine/storage/snapshot/pom.xml

@@ -56,6 +56,11 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

framework/db/pom.xml

@@ -40,6 +40,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-dbcp2</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+        </dependency>
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>
@@ -48,6 +52,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-pool2</artifactId>
         </dependency>
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-utils</artifactId>

framework/db/src/main/java/com/cloud/utils/db/ConnectionConcierge.java

@@ -145,7 +145,7 @@ public void unregister(String name) {
         protected String testValidity(String name, Connection conn) {
             if (conn != null) {
                 synchronized (conn) {
-                    try (PreparedStatement pstmt = conn.prepareStatement("SELECT 1");) {
+                    try (PreparedStatement pstmt = conn.prepareStatement("/* ping */ SELECT 1");) {
                         pstmt.executeQuery();
                     } catch (Throwable th) {
                         logger.error("Unable to keep the db connection for " + name, th);