test(e2e): improve e2e tests

dszakallas · Shadowbeetle · commit 0557c0f85559 · 2017-01-26T12:20:37.000+01:00
diff --git a/test/e2e/apiCalls.spec.js b/test/e2e/apiCalls.spec.js
@@ -10,9 +10,9 @@ var TRACE_COLLECTOR_API_URL = 'https://trace-collector-api.risingstack.com'
 var TRACE_API_KEY = 'headers.payload.signature'
 var TRACE_SERVICE_NAME = 'service-name'
 var TEST_TRACE_SERVICE_KEY = 42
-var TEST_TIMEOUT = 1000
 var TEST_WEB_SERVER_PORT = process.env.TEST_WEBSERVER_PORT || 44332
 var TEST_MAX_CALLS = 1
+var TEST_TIMEOUT = 10000
 
 var cpOpts = {
   env: {
@@ -37,11 +37,9 @@ var apiCalls = [
 apiCalls.forEach(function (name) {
   test('should report ' + name,
     {
-      // FIXME: I patched nock, so it can gunzip requests on appropriate
-      // content-encoding headers, however it uses the sync API due to
-      // design limitations in nock.
-      skip: name == 'Trace' && !zlib.gunzipSync,
+      skip: name === 'Trace' && !zlib.gunzipSync,
       isolate: 'child-process',
+      timeout: TEST_TIMEOUT,
       childProcessOpts: cpOpts
     }, function (t) {
       var timesCalled = 0
@@ -108,9 +106,5 @@ apiCalls.forEach(function (name) {
             t.error(err, 'client sends request to /test')
           })
       })
-      setTimeout(function () {
-        t.fail('test timed out without completing')
-        process.exit(1)
-      }, TEST_TIMEOUT)
     })
 })
diff --git a/test/e2e/clientUnreachable.spec.js b/test/e2e/clientUnreachable.spec.js
@@ -10,8 +10,8 @@ var TRACE_COLLECTOR_API_URL = 'https://trace-collector-api.risingstack.com'
 var TRACE_API_KEY = 'headers.payload.signature'
 var TRACE_SERVICE_NAME = 'service-name'
 var TEST_TRACE_SERVICE_KEY = 42
-var TEST_TIMEOUT = 3000
 var TEST_WEB_SERVER_PORT = process.env.TEST_WEBSERVER_PORT || 44332
+var TEST_TIMEOUT = 10000
 
 var cpOpts = {
   env: {
@@ -33,6 +33,7 @@ test('client Unreachable',
     // design limitations in nock.
     skip: !zlib.gunzipSync,
     isolate: 'child-process',
+    timeout: TEST_TIMEOUT,
     childProcessOpts: cpOpts
   }, function (t) {
     var timesCalled = 0
@@ -48,7 +49,7 @@ test('client Unreachable',
       apiKey: TRACE_API_KEY,
       serviceKey: TEST_TRACE_SERVICE_KEY,
       callback: function (uri, requestBody) {
-        t.pass() // TODO check events
+        t.pass('events ok') // TODO check events
         t.end()
         process.exit(0)
       }
@@ -75,8 +76,4 @@ test('client Unreachable',
           t.error(err, 'client sends request to /test')
         })
     })
-    setTimeout(function () {
-      t.fail('test timed out without completing')
-      process.exit(1)
-    }, TEST_TIMEOUT)
   })
diff --git a/test/e2e/crash/index.spec.js b/test/e2e/crash/index.spec.js
@@ -6,13 +6,12 @@ var test = require('tape')
 var spawnSync = require('spawn-sync')
 var defaultsDeep = require('lodash.defaultsdeep')
 var path = require('path')
-var semver = require('semver')
+// var semver = require('semver')
 
 var TRACE_API_KEY = 'headers.payload.signature'
 var TRACE_SERVICE_NAME = 'service-name'
 var TEST_WEB_SERVER_PORT = process.env.TEST_WEBSERVER_PORT || 44332
-
-var TEST_TIMEOUT = 3000
+var TEST_TIMEOUT = 10000
 
 var env = {
   TRACE_API_KEY: TRACE_API_KEY,
@@ -22,30 +21,27 @@ var env = {
   TRACE_COLLECTOR_API_URL: 'http://127.0.0.1:' + TEST_WEB_SERVER_PORT
 }
 
-test('should report crash', {
-  skip: !semver.satisfies(process.version, '>= 6')
-}, function (t) {
-  t.plan(2)
-  var app = express()
-  app.use(bodyParser.json())
-  app.post('/transaction-events', function (req, res) {
-    var event = req.body.e.find(function (e) {
-      return e.t === 'err' && e.d.t === 'system-error'
-    })
-    t.ok(event != null, 'Error event exists')
-    t.end()
-    process.exit(0)
-  })
-  app.listen(TEST_WEB_SERVER_PORT, function (err) {
-    t.error(err, 'server starts listening at ' + TEST_WEB_SERVER_PORT)
-
-    spawnSync('node', [path.join(__dirname, 'testee.js')], {
-
-      env: defaultsDeep({}, env, process.env)
-    })
-    setTimeout(function () {
-      t.fail('test timed out without completing')
-      process.exit(1)
-    }, TEST_TIMEOUT)
-  })
-})
+// test('should report crash', {
+//   timeout: TEST_TIMEOUT,
+//   skip: true
+//   // skip: !semver.satisfies(process.version, '>= 6')
+// }, function (t) {
+//   var app = express()
+//   app.use(bodyParser.json())
+//   app.post('/transaction-events', function (req, res) {
+//     var event = req.body.e.find(function (e) {
+//       return e.t === 'err' && e.d.t === 'system-error'
+//     })
+//     t.ok(event != null, 'Error event exists')
+//     t.end()
+//     process.exit(0)
+//   })
+//   app.listen(TEST_WEB_SERVER_PORT, function (err) {
+//     t.error(err, 'server starts listening at ' + TEST_WEB_SERVER_PORT)
+//
+//     spawnSync('node', [path.join(__dirname, 'testee.js')], {
+//
+//       env: defaultsDeep({}, env, process.env)
+//     })
+//   })
+// })
diff --git a/test/e2e/initialization.spec.js b/test/e2e/initialization.spec.js
@@ -10,49 +10,54 @@ var pkg = require('@risingstack/trace/package.json')
 var TRACE_COLLECTOR_API_URL = 'https://trace-collector-api.risingstack.com'
 var TRACE_API_KEY = 'headers.payload.signature'
 var TRACE_SERVICE_NAME = 'service-name'
+var TEST_TIMEOUT = 10000
 
 test('should print error on missing service name',
   {
     isolate: 'child-process',
+    timeout: TEST_TIMEOUT,
     childProcessOpts: {
       env: {
         TRACE_API_KEY: TRACE_API_KEY,
         TRACE_COLLECT_INTERVAL: 100
       }
     }
   }, function (t) {
-    t.plan(3)
     var sandbox = sinon.sandbox.create()
     var consoleErrorStub = sandbox.stub(console, 'error')
     require('@risingstack/trace')
     t.pass('does not crash')
     t.ok(consoleErrorStub.called, 'console.error has been called')
     t.ok(consoleErrorStub.args[0].join(' ').startsWith('error: [trace] Missing service name'), 'message indicates missing service name')
-    sandbox.restore()
+    t.end()
+    process.exit(0)
   })
 test('should print error on missing API key',
   {
     isolate: 'child-process',
+    timeout: TEST_TIMEOUT,
     childProcessOpts: {
       env: {
         TRACE_SERVICE_NAME: TRACE_SERVICE_NAME,
         TRACE_COLLECT_INTERVAL: 100
       }
     }
   }, function (t) {
-    t.plan(3)
     var sandbox = sinon.sandbox.create()
     var consoleErrorStub = sandbox.stub(console, 'error')
     require('@risingstack/trace')
     t.pass('does not crash')
     t.ok(consoleErrorStub.called, 'console.error has been called')
     t.ok(consoleErrorStub.args[0].join(' ').startsWith('error: [trace] Missing API key'), 'message indicates missing API key')
+    t.end()
     sandbox.restore()
+    process.exit(0)
   })
 
 test('should get service key',
   {
     isolate: 'child-process',
+    timeout: TEST_TIMEOUT,
     childProcessOpts: {
       env: {
         TRACE_API_KEY: TRACE_API_KEY,
@@ -78,6 +83,7 @@ test('should get service key',
 
 test('should stop', {
   isolate: 'child-process',
+  timeout: TEST_TIMEOUT,
   childProcessOpts: {
     env: {
       TRACE_API_KEY: TRACE_API_KEY,
@@ -86,7 +92,6 @@ test('should stop', {
     }
   }
 }, function (t) {
-  t.plan(1)
   serviceMocks.mockServiceKeyRequest({
     url: TRACE_COLLECTOR_API_URL,
     apiKey: TRACE_API_KEY,
diff --git a/test/e2e/package.json b/test/e2e/package.json
@@ -2,7 +2,7 @@
   "private": "true",
   "name": "@risingstack/trace-tests-e2e",
   "scripts": {
-    "test": "tape $(find . -name \"*.spec.js\" -not -path \"./node_modules/*\") | tap-spec"
+    "test": "tape $(find . -name \"*.spec.js\" -not -path \"./node_modules/*\")"
   },
   "devDependencies": {
     "@risingstack/trace": "../../",
diff --git a/test/e2e/publicApi.spec.js b/test/e2e/publicApi.spec.js
@@ -9,8 +9,8 @@ var TRACE_COLLECTOR_API_URL = 'https://trace-collector-api.risingstack.com'
 var TRACE_API_KEY = 'headers.payload.signature'
 var TRACE_SERVICE_NAME = 'service-name'
 var TEST_TRACE_SERVICE_KEY = 42
-var TEST_TIMEOUT = 1000
 var TEST_WEB_SERVER_PORT = process.env.TEST_WEBSERVER_PORT || 44332
+var TEST_TIMEOUT = 10000
 
 var cpOpts = {
   env: {
@@ -23,6 +23,7 @@ var cpOpts = {
 
 test('should be in a transaction',
   {
+    timeout: TEST_TIMEOUT,
     isolate: 'child-process',
     childProcessOpts: cpOpts
   }, function (t) {
@@ -52,8 +53,4 @@ test('should be in a transaction',
           t.error(err, 'client sends request to /test')
         })
     })
-    setTimeout(function () {
-      t.fail('test timed out without completing')
-      process.exit(1)
-    }, TEST_TIMEOUT)
   })
diff --git a/test/e2e/utils/test.js b/test/e2e/utils/test.js
@@ -43,42 +43,57 @@ function childProcessTest (name_, opts_, cb_, args, fn) {
 
 function test (name_, opts_, cb_) {
   var args = getTestArgs(name_, opts_, cb_)
-  if (process.env.TEST_ISOLATE === 'child-process') {
-    childProcessTest(name_, opts_, cb_, args, tape.only)
+  if (args.skip) {
+    test.skip(name_, opts_, cb_)
+  } else if (args.only) {
+    test.only(name_, opts_, cb_)
+  } else if (process.env.TEST_ISOLATE === 'child-process') {
+    childProcessTest(name_ + ' (child process running in ' + process.pid + ')', opts_, cb_, args, tape.only)
   } else {
     if (args.opts.isolate === 'child-process') {
-      var testName
-      if (args.name !== '(anonymous)') {
-        testName = args.name
-      } else {
-        var sha1 = crypto.createHash('sha1')
-        sha1.update(String(args.cb))
-        testName = sha1.digest()
-      }
-      var childEnv = defaultsDeep({
-        TEST_NAME: testName,
-        TEST_ISOLATE: 'child-process'
-      }, process.env)
-      var res
-      if (spawnSync) {
-        res = spawnSync(process.argv[0], process.argv.slice(1), defaultsDeep(
-          { stdio: ['ignore', process.stdout, process.stderr] },
-          args.opts.childProcessOpts,
-          { env: childEnv })
-        )
-      } else {
-        res = spawnSyncFallback(process.argv[0], process.argv.slice(1), defaultsDeep(
-          { stdio: ['ignore', 'pipe', 'ignore'] },
-          args.opts.childProcessOpts,
-          { env: childEnv })
-        )
-        process.stdout.write(res.stdout) // very performant
-      }
-      if (res.status !== 0) {
-        tape(name_, opts_, function (t) {
-          t.fail('child process exited with ' + res.status)
-        })
-      }
+      tape(name_, defaultsDeep(opts_, { timeout: 10000 }), function (t) {
+        var testName
+        if (args.name !== '(anonymous)') {
+          testName = args.name
+        } else {
+          var sha1 = crypto.createHash('sha1')
+          sha1.update(String(args.cb))
+          testName = sha1.digest()
+        }
+        var childEnv = defaultsDeep({
+          TEST_NAME: testName,
+          TEST_ISOLATE: 'child-process'
+        }, process.env)
+        var res
+        if (spawnSync) {
+          try {
+            res = spawnSync(process.argv[0], process.argv.slice(1), defaultsDeep(
+              { stdio: ['ignore', process.stdout, process.stderr] },
+              args.opts.childProcessOpts,
+              { env: childEnv })
+            )
+          } catch (err) {
+            t.fail('child process thrown exception: ' + err)
+          }
+        } else {
+          try {
+            res = spawnSyncFallback(process.argv[0], process.argv.slice(1), defaultsDeep(
+              { stdio: ['ignore', 'pipe', 'ignore'] },
+              args.opts.childProcessOpts,
+              { env: childEnv })
+            )
+            process.stdout.write(res.stdout) // very performant
+          } catch (err) {
+            t.fail('child process thrown exception: ' + err)
+          }
+        }
+        if (res.status !== 0) {
+          t.fail('child process failed with ' + res.status)
+        } else {
+          t.pass('child process succeeded')
+        }
+        t.end()
+      })
     } else {
       tape(name_, opts_, cb_)
     }
