chore(tracer): add option to configure sampler

Author: dszakallas

lib/agent/tracer/collector.js

@@ -7,7 +7,7 @@ var uuid = require('node-uuid')
 var assign = require('lodash.assign')
 var levels = require('./severity')
 var some = require('lodash.some')
-var ReservoirSampler = require('./reservoirSampler')
+var reservoirSampler = require('./samplers/reservoir')
 
 var EVENT_TYPE = {
   CLIENT_SEND: 'cs',
@@ -39,13 +39,13 @@ function Collector (options) {
   this.serviceKey = options.serviceKey
   this.mustCollectSeverity = levels.ERROR
   this.defaultSeverity = levels.INFO
-  this.samplerLimit = options.samplerLimit || 100
+  this.samplerLimit = options.samplerLimit
 
   // init required variables
   this._noStack = options.noStack
   this._eventTtl = options.eventTtl || 1
   this._eventBuffers = { }
-  this._sampler = new ReservoirSampler(this.samplerLimit)
+  this._sampler = reservoirSampler.create(this.samplerLimit)
 }
 
 Collector.prototype.LEVELS = levels

lib/agent/tracer/collector.spec.js

@@ -746,8 +746,8 @@ describe('Collector', function () {
     })
   })
 
-  describe('Sampling', function () {
-    it('happens when there is a big amount of data #1', function () {
+  describe('Sampler', function () {
+    it('samples big amount of data #1', function () {
       this.sandbox.stub(microtime, 'now', function () {
         return 2
       })
@@ -780,7 +780,7 @@ describe('Collector', function () {
       expect(length).to.be.above(0)
     })
 
-    it('happens when there is a big amount of data #2', function () {
+    it('happens big amount of data #2', function () {
       this.sandbox.stub(microtime, 'now', function () {
         return 2
       })
@@ -802,5 +802,30 @@ describe('Collector', function () {
       expect(length).to.be.below(50)
       expect(length).to.be.above(0)
     })
+
+    it('is emptied', function () {
+      this.sandbox.stub(microtime, 'now', function () {
+        return 2
+      })
+      var tracer = new Collector(assign({ samplerLimit: 10 }, options))
+      function report () {
+        var cs = {
+          payload: { severity: tracer.mustCollectSeverity },
+          briefcase: { }
+        }
+        var csResult = tracer.clientSend(cs.payload, cs.briefcase)
+
+        var cr = {
+          payload: { }
+        }
+        tracer.clientRecv(cr.payload, {}, csResult.briefcase)
+      }
+      for (var i = 0; i < 50; ++i) { report() }
+      var length = tracer.collect().length
+      expect(length).to.be.below(50)
+      expect(length).to.be.above(0)
+
+      expect(tracer._sampler.size()).to.equal(0)
+    })
   })
 })

lib/agent/tracer/index.js

@@ -18,7 +18,8 @@ function Tracer (options) {
   // init required variables
   this.collector = new Collector({
     eventTtl: 3,
-    noStack: options.config.disableStackTrace
+    noStack: options.config.disableStackTrace,
+    samplerLimit: options.config.samplerLimit
   })
   this._api = options.collectorApi
 

lib/agent/tracer/reservoirSampler.js lib/agent/tracer/samplers/reservoir.jslib/agent/tracer/reservoirSampler.js renamed to lib/agent/tracer/samplers/reservoir.js

@@ -1,12 +1,12 @@
 
-function Reservoir (limit) {
-  this.MAX_ITEMS = limit || 10
+function ReservoirSampler (limit) {
+  this.MAX_ITEMS = limit || Infinity
 
   this._itemsSeen = 0
   this._data = []
 }
 
-Reservoir.prototype.add = function (item) {
+ReservoirSampler.prototype.add = function (item) {
   var inserted = true
   if (this._itemsSeen < this.MAX_ITEMS) {
     this._data.push(item)
@@ -18,14 +18,18 @@ Reservoir.prototype.add = function (item) {
   return inserted
 }
 
-Reservoir.prototype.flush = function () {
+ReservoirSampler.prototype.flush = function () {
   var data = this._data
   this._data = []
   this._itemsSeen = 0
   return data
 }
 
-Reservoir.prototype._replace = function (item) {
+ReservoirSampler.prototype.size = function () {
+  return this._data.length
+}
+
+ReservoirSampler.prototype._replace = function (item) {
   var toReplaceIndex = Math.floor(Math.random() * (this._itemsSeen + 2))
   if (toReplaceIndex < this.MAX_ITEMS) {
     this._data[toReplaceIndex] = item
@@ -34,4 +38,6 @@ Reservoir.prototype._replace = function (item) {
   return false
 }
 
-module.exports = Reservoir
+module.exports.create = function (limit) {
+  return new ReservoirSampler(limit)
+}

lib/agent/tracer/reservoirSampler.spec.js …/agent/tracer/samplers/reservoir.spec.jslib/agent/tracer/reservoirSampler.spec.js renamed to lib/agent/tracer/samplers/reservoir.spec.js lib/agent/tracer/reservoirSampler.spec.js renamed to lib/agent/tracer/samplers/reservoir.spec.js

@@ -1,5 +1,5 @@
 var expect = require('chai').expect
-var ReservoirSampler = require('./reservoirSampler')
+var reservoirSampler = require('./reservoir')
 
 describe('The Reservoir Sampler', function () {
   var itemToAdd = {
@@ -8,7 +8,7 @@ describe('The Reservoir Sampler', function () {
 
   describe('when seen count is below the size limit', function () {
     it('adds the item', function () {
-      var sampler = new ReservoirSampler(3)
+      var sampler = reservoirSampler.create(3)
       var isAdded = sampler.add(itemToAdd)
 
       expect(isAdded).to.eql(true)
@@ -21,7 +21,7 @@ describe('The Reservoir Sampler', function () {
       this.sandbox.stub(Math, 'random', function () {
         return 0
       })
-      var sampler = new ReservoirSampler(1)
+      var sampler = reservoirSampler.create(1)
       sampler._itemsSeen = 1
       var isAdded = sampler.add(itemToAdd)
       expect(isAdded).to.eql(true)
@@ -31,7 +31,7 @@ describe('The Reservoir Sampler', function () {
       this.sandbox.stub(Math, 'random', function () {
         return 1
       })
-      var sampler = new ReservoirSampler(1)
+      var sampler = reservoirSampler.create(1)
       sampler._itemsSeen = 1
       var isAdded = sampler.add(itemToAdd)
       expect(isAdded).to.eql(false)

lib/config.js

@@ -4,6 +4,8 @@ config.collectInterval = 2 * 60 * 1000
 config.updateInterval = 15 * 1000
 config.healthcheckInterval = 10 * 1000
 
+config.samplerLimit = 100
+
 config.collectorLanguage = 'nodejs'
 config.collectorApiUrl = 'https://trace-collector-api.risingstack.com'
 config.collectorApiSampleEndpoint = '/transaction-events'

lib/utils/configReader.js

@@ -67,6 +67,11 @@ ConfigReader.prototype._getEnvVarConfig = function () {
     keepQueryParams: process.env.TRACE_KEEP_QUERY_PARAMS
   }
 
+  if (process.env.TRACE_SAMPLER_LIMIT) {
+    var samplerLimit = Number(process.env.TRACE_SAMPLER_LIMIT)
+    !isNaN(samplerLimit) && defaults(envVarConfig, { samplerLimit: samplerLimit })
+  }
+
   var ignoreHeaders
   var ignoreOutgoingHosts
 

lib/utils/configReader.spec.js

@@ -345,6 +345,39 @@ describe('Config Reader module', function () {
   })
 
   describe('config variable', function () {
+    describe('samplerLimit', function () {
+      it('should be configurable with by env var', function () {
+        this.sandbox.stub(process, 'env', {
+          TRACE_SAMPLER_LIMIT: 1000
+        })
+
+        var configReader = ConfigReader.create({
+          apiKey: testApiToken,
+          reporter: 'dummy',
+          serviceName: 'test'
+        })
+
+        var config = configReader.getConfig()
+
+        expect(config.samplerLimit).to.equal(1000)
+      })
+      it('should be tolerant', function () {
+        this.sandbox.stub(process, 'env', {
+          TRACE_SAMPLER_LIMIT: 'asdf'
+        })
+
+        var configReader = ConfigReader.create({
+          apiKey: testApiToken,
+          reporter: 'dummy',
+          serviceName: 'test'
+        })
+
+        var config = configReader.getConfig()
+
+        expect(config.samplerLimit).not.to.be.a('string')
+        expect(config.samplerLimit).not.to.be.NaN
+      })
+    })
     describe('traceProxy', function () {
       it('should be configurable with the HTTP_PROXY env var', function () {
         this.sandbox.stub(process, 'env', {