make sure we error out when CRT result is smaller than netty

Author: TingDaoK

.builder/actions/localhost_canary.py

@@ -10,7 +10,8 @@ def run(self, env):
         actions = [
             "mvn install -DskipTests",
             "cd ./samples/HttpClientCanary && mvn install",
-            "cd ./samples/HttpClientCanary && mvn exec:exec@netty exec:exec@crt"
+            "cd ./samples/HttpClientCanary && mvn exec:exec@netty",
+            "cd ./samples/HttpClientCanary && mvn exec:exec@crt"
         ]
 
         return Builder.Script(actions, name='aws-crt-java-test')

samples/HttpClientCanary/src/main/java/com/canary/CanaryUtils.java

@@ -54,11 +54,13 @@ public static double calculateSTD(ArrayList<Double> list) {
         return Math.sqrt(variance);
      }
 
-     public static void printResult(ArrayList<Double> list) {
+    /* Return avg of the list and print out the avg and std */
+     public static double printResult(ArrayList<Double> list) {
          double avg = calculateAverage(list);
          double std = calculateSTD(list);
          System.out.println("Result collected has: " + list.size());
          System.out.println("avg of all samples: " + avg);
          System.out.println("Standard deviation of all samples: " + std);
+         return avg;
       }
 }

samples/HttpClientCanary/src/main/java/com/canary/Http2StreamManagerCanary.java

@@ -48,6 +48,7 @@ public class Http2StreamManagerCanary {
     private int maxStreams = 100;
     private int maxConnections = 50;
     private int batchNum;
+    private String nettyResultPath;
 
     private Http2StreamManager createStreamManager(URI uri, int numConnections) {
 
@@ -65,7 +66,7 @@ private Http2StreamManager createStreamManager(URI uri, int numConnections) {
                     .withSocketOptions(sockOpts)
                     .withUri(uri)
                     .withMaxConnections(numConnections);
-            if(useTls) {
+            if (useTls) {
                 connectionManagerOptions.withTlsContext(tlsContext);
             } else {
                 options.withPriorKnowledge(true);
@@ -165,7 +166,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) {
 
                         opts.incrementAndGet();
                         int requestCompletedNum = requestCompleted.incrementAndGet();
-                        if(requestCompletedNum == concurrentNum) {
+                        if (requestCompletedNum == concurrentNum) {
                             requestCompleteFuture.complete(null);
                         }
                     }
@@ -180,34 +181,53 @@ private void runCanary(int warmupLoops, int loops, long timerSecs) throws Except
         ArrayList<Double> warmupResults = new ArrayList<>();
         ArrayList<Double> results = new ArrayList<>();
         AtomicInteger streamFailed = new AtomicInteger(0);
-        // Log.initLoggingToFile(Log.LogLevel.Error, "errorlog.txt");
 
-        System.out.println("batchNum: "+ this.batchNum);
-        System.out.println("maxStreams: "+ this.maxStreams);
-        System.out.println("maxConnections: "+ this.maxConnections);
+        System.out.println("batchNum: " + this.batchNum);
+        System.out.println("maxStreams: " + this.maxStreams);
+        System.out.println("maxConnections: " + this.maxConnections);
         try (Http2StreamManager streamManager = createStreamManager(uri, this.maxConnections)) {
             AtomicInteger opts = new AtomicInteger(0);
             AtomicBoolean done = new AtomicBoolean(false);
-            ScheduledExecutorService scheduler = createDataCollector(warmupLoops, loops, timerSecs, opts, done, warmupResults, results);
+            ScheduledExecutorService scheduler = createDataCollector(warmupLoops, loops, timerSecs, opts, done,
+                    warmupResults, results);
             concurrentRequests(streamManager, batchNum, streamFailed, opts, done);
             scheduler.shutdown();
         }
         System.out.println("Failed request num: " + streamFailed.get());
         System.out.println("////////////// warmup results //////////////");
         printResult(warmupResults);
         System.out.println("////////////// real results //////////////");
-        printResult(results);
+        double avg_result = printResult(results);
+        BufferedReader reader = new BufferedReader(new FileReader(nettyResultPath));
+        StringBuilder stringBuilder = new StringBuilder();
+        String line = null;
+        while ((line = reader.readLine()) != null) {
+            stringBuilder.append(line);
+        }
+        reader.close();
+        stringBuilder.deleteCharAt(stringBuilder.length() - 1);
+        String content = stringBuilder.toString();
+        double netty_result = Double.parseDouble(content);
+        if (avg_result < netty_result) {
+            System.out.println("CRT result is smaller than netty. CRT: " + avg_result + " Netty: " + netty_result);
+            System.exit(-1);
+        }
+
         CrtResource.logNativeResources();
         CrtResource.waitForNoResources();
+
     }
 
     public static void main(String[] args) throws Exception {
         System.out.println("Current JVM version - " + System.getProperty("java.version"));
-        /* TODO: make all those number configurable */
+
         Http2StreamManagerCanary canary = new Http2StreamManagerCanary();
-        canary.uri = new URI("https://localhost:8443/echo");
-        canary.maxConnections = 8;
-        canary.maxStreams = 20;
+
+        canary.uri = new URI(System.getProperty("aws.crt.http.canary.uri", "https://localhost:8443/echo"));
+        canary.maxConnections = Integer.parseInt(System.getProperty("aws.crt.http.canary.maxConnections", "8"));
+        canary.maxStreams = Integer.parseInt(System.getProperty("aws.crt.http.canary.maxStreams", "20"));
+        canary.nettyResultPath = System.getProperty("aws.crt.http.canary.nettyResultPath", "netty_result.txt");
+
         canary.batchNum = canary.maxStreams * canary.maxConnections;
         canary.runCanary(5, 5, 30);
     }

samples/HttpClientCanary/src/main/java/com/canary/SDKNettyClientCanary.java

@@ -13,6 +13,8 @@
 import software.amazon.awssdk.utils.AttributeMap;
 import software.amazon.awssdk.http.nio.netty.Http2Configuration;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
@@ -31,6 +33,7 @@ public class SDKNettyClientCanary {
     private int batchNum;
     private int maxStreams = 100;
     private int maxConnections = 50;
+    private String nettyResultPath;
 
     public static AttributeMap.Builder trustAllTlsAttributeMapBuilder() {
         return AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, true);
@@ -145,15 +148,20 @@ private void runCanary(int warmupLoops, int loops, long timerSecs) throws Except
         System.out.println("////////////// warmup results //////////////");
         printResult(warmupResults);
         System.out.println("////////////// real results //////////////");
-        printResult(results);
+        double avg_result = printResult(results);
+        BufferedWriter writer = new BufferedWriter(new FileWriter(this.nettyResultPath, false));
+        writer.append(Double.toString(avg_result));
+        writer.close();
     }
 
     public static void main(String[] args) throws Exception {
         SDKNettyClientCanary canary = new SDKNettyClientCanary();
-        /* TODO: make all those number configurable */
-        canary.uri = new URI("https://localhost:8443/echo");
-        canary.maxConnections = 8;
-        canary.maxStreams = 20;
+
+        canary.uri = new URI(System.getProperty("aws.crt.http.canary.uri", "https://localhost:8443/echo"));
+        canary.maxConnections = Integer.parseInt(System.getProperty("aws.crt.http.canary.maxConnections", "8"));
+        canary.maxStreams = Integer.parseInt(System.getProperty("aws.crt.http.canary.maxStreams", "20"));
+        canary.nettyResultPath = System.getProperty("aws.crt.http.canary.nettyResultPath", "netty_result.txt");
+
         canary.batchNum = canary.maxStreams * canary.maxConnections;
         canary.runCanary(5, 5, 30);
     }