bench(sirun): report operations per second

benchmark/sirun/child_process/index.js

@@ -18,7 +18,7 @@ require('../../../packages/datadog-instrumentations/src/child_process')
 const channel = dc.tracingChannel('datadog:child_process:execution')
 
 const { VARIANT } = process.env
-const ITERATIONS = Number(process.env.ITERATIONS) || 5_000_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 // execFile-shaped (file + array args) vs exec-shaped (single shell string).
 const useFileArgs = VARIANT === 'file-args'
@@ -91,7 +91,7 @@ assert.ok(probeContext.abortController, 'context is missing the AbortController'
 
 let sink = 0
 guard.loopStart()
-for (let i = 0; i < ITERATIONS; i++) {
+for (let i = 0; i < OPERATIONS; i++) {
   sink += instrumentedCall()
 }
 guard.done()

benchmark/sirun/child_process/meta.json

@@ -7,10 +7,10 @@
   "instructions": true,
   "variants": {
     "shell-string": {
-      "env": { "VARIANT": "shell-string", "ITERATIONS": "1100000" }
+      "env": { "VARIANT": "shell-string", "OPERATIONS": "1100000" }
     },
     "file-args": {
-      "env": { "VARIANT": "file-args", "ITERATIONS": "1000000" }
+      "env": { "VARIANT": "file-args", "OPERATIONS": "1000000" }
     }
   }
 }

benchmark/sirun/collect-overview.js

@@ -54,7 +54,7 @@ function categoryOf (name) {
 }
 
 function innerCount (env = {}) {
-  return env.COUNT || env.ITERATIONS || env.QUERIES || (env.REQS ? `${env.REQS} reqs` : '-')
+  return env.OPERATIONS || env.COUNT || (env.REQS ? `${env.REQS} reqs` : '-')
 }
 
 function stats (times) {

benchmark/sirun/datastreams/index.js

@@ -14,7 +14,7 @@ const { getMessageSize } = require('../../../packages/dd-trace/src/datastreams/s
 
 const { VARIANT } = process.env
 
-const ITERATIONS = 1_200_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 const processor = new DataStreamsProcessor({
   dsmEnabled: true,
@@ -107,13 +107,13 @@ assert.ok(CONSUME_CARRIERS[0]['dd-pathway-ctx-base64'], 'codec did not inject pa
 
 guard.loopStart()
 if (VARIANT === 'consume') {
-  for (let iteration = 0; iteration < ITERATIONS; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     const carrier = CONSUME_CARRIERS[iteration % CONSUME_CARRIERS.length]
     const ctx = DsmPathwayCodec.decode(carrier)
     processor.setCheckpoint(CONSUMER_TAGS[iteration % CONSUMER_TAGS.length], span, ctx, MESSAGE_SIZE)
   }
 } else if (VARIANT === 'produce-with-message-size') {
-  for (let iteration = 0; iteration < ITERATIONS; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     const payloadSize = getMessageSize(MESSAGE)
     const ctx = processor.setCheckpoint(
       PRODUCER_TAGS[iteration % PRODUCER_TAGS.length],
@@ -124,7 +124,7 @@ if (VARIANT === 'consume') {
     DsmPathwayCodec.encode(ctx, {})
   }
 } else if (VARIANT === 'produce-manual-checkpoint') {
-  for (let iteration = 0; iteration < ITERATIONS; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     const ctx = processor.setCheckpoint(
       MANUAL_PRODUCER_TAGS[iteration % MANUAL_PRODUCER_TAGS.length],
       span,
@@ -134,7 +134,7 @@ if (VARIANT === 'consume') {
     DsmPathwayCodec.encode(ctx, {})
   }
 } else if (VARIANT === 'produce-high-cardinality') {
-  for (let iteration = 0; iteration < ITERATIONS; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     const ctx = processor.setCheckpoint(
       HIGH_CARDINALITY_PRODUCER_TAGS[iteration % HIGH_CARDINALITY_PRODUCER_TAGS.length],
       span,
@@ -144,7 +144,7 @@ if (VARIANT === 'consume') {
     DsmPathwayCodec.encode(ctx, {})
   }
 } else {
-  for (let iteration = 0; iteration < ITERATIONS; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     const ctx = processor.setCheckpoint(
       PRODUCER_TAGS[iteration % PRODUCER_TAGS.length],
       span,

benchmark/sirun/datastreams/meta.json

@@ -6,10 +6,10 @@
   "iterations": 20,
   "instructions": true,
   "variants": {
-    "produce": { "env": { "VARIANT": "produce" } },
-    "consume": { "env": { "VARIANT": "consume" } },
-    "produce-with-message-size": { "env": { "VARIANT": "produce-with-message-size" } },
-    "produce-manual-checkpoint": { "env": { "VARIANT": "produce-manual-checkpoint" } },
-    "produce-high-cardinality": { "env": { "VARIANT": "produce-high-cardinality" } }
+    "produce": { "env": { "VARIANT": "produce", "OPERATIONS": "1200000" } },
+    "consume": { "env": { "VARIANT": "consume", "OPERATIONS": "1200000" } },
+    "produce-with-message-size": { "env": { "VARIANT": "produce-with-message-size", "OPERATIONS": "1200000" } },
+    "produce-manual-checkpoint": { "env": { "VARIANT": "produce-manual-checkpoint", "OPERATIONS": "1200000" } },
+    "produce-high-cardinality": { "env": { "VARIANT": "produce-high-cardinality", "OPERATIONS": "1200000" } }
   }
 }

benchmark/sirun/debugger/app.js

@@ -6,7 +6,7 @@
 
 const guard = require('../startup-guard')
 
-const ITERATIONS = Number(process.env.ITERATIONS) || 5000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 if (process.env.DD_DYNAMIC_INSTRUMENTATION_ENABLED === 'true') {
   // The devtools worker and its ports are unref'd, so nothing holds the event
@@ -26,7 +26,7 @@ if (process.env.DD_DYNAMIC_INSTRUMENTATION_ENABLED === 'true') {
 
 function runWork () {
   guard.loopStart()
-  for (let i = 0; i < ITERATIONS; i++) {
+  for (let i = 0; i < OPERATIONS; i++) {
     doSomeWork(i)
   }
   guard.done(0.35)

benchmark/sirun/debugger/meta.json

@@ -25,7 +25,7 @@
         "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true",
         "BREAKPOINT_FILE": "app.js",
         "BREAKPOINT_LINE": "37",
-        "ITERATIONS": "3500"
+        "OPERATIONS": "3500"
       }
     },
     "line-probe-with-snapshot-default": {
@@ -40,7 +40,7 @@
         "BREAKPOINT_LINE": "37",
         "CAPTURE_SNAPSHOT": "true",
         "MAX_SNAPSHOTS_PER_SECOND_GLOBALLY": "1000000",
-        "ITERATIONS": "2450"
+        "OPERATIONS": "2450"
       }
     },
     "line-probe-with-snapshot-minimal": {
@@ -59,7 +59,7 @@
         "MAX_COLLECTION_SIZE": "0",
         "MAX_FIELD_COUNT": "0",
         "MAX_LENGTH": "9007199254740991",
-        "ITERATIONS": "3500"
+        "OPERATIONS": "3500"
       }
     }
   }

benchmark/sirun/dogstatsd/index.js

@@ -6,7 +6,7 @@ const guard = require('../startup-guard')
 const { DogStatsDClient, MetricsAggregationClient } = require('../../../packages/dd-trace/src/dogstatsd')
 
 const { VARIANT } = process.env
-const ITERATIONS = Number(process.env.ITERATIONS) || 5_000_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 // Every metric the tracer emits (runtime metrics, custom metrics) runs through
 // DogStatsDClient._add: build the `stat:value|type` line, splice the global and
@@ -48,7 +48,7 @@ if (VARIANT === 'aggregated') {
   // The runtime-metrics path: accumulate into the tag tree, then flush walks the
   // tree and formats every node through the client. Stubbed socket on flush.
   const agg = new MetricsAggregationClient(client)
-  for (let i = 0; i < ITERATIONS; i++) {
+  for (let i = 0; i < OPERATIONS; i++) {
     agg.count(NAME, 1, FEW_TAGS)
     agg.gauge('runtime.node.mem.heap_used', i, FEW_TAGS)
     if ((i & 0x3FF) === 0) agg.flush()
@@ -57,7 +57,7 @@ if (VARIANT === 'aggregated') {
 } else {
   const tags = VARIANT === 'no-tags' ? undefined : (VARIANT === 'many-tags' ? MANY_TAGS : FEW_TAGS)
   const type = VARIANT === 'no-tags' ? 'c' : 'g'
-  for (let i = 0; i < ITERATIONS; i++) {
+  for (let i = 0; i < OPERATIONS; i++) {
     client._add(NAME, i, type, tags)
     // Drain the datagram queue without sending so memory stays flat.
     if ((i & 0x7FF) === 0) client._queue.length = 0

benchmark/sirun/dogstatsd/meta.json

@@ -7,16 +7,16 @@
   "instructions": true,
   "variants": {
     "no-tags": {
-      "env": { "VARIANT": "no-tags", "ITERATIONS": "6000000" }
+      "env": { "VARIANT": "no-tags", "OPERATIONS": "6000000" }
     },
     "with-tags": {
-      "env": { "VARIANT": "with-tags", "ITERATIONS": "8300000" }
+      "env": { "VARIANT": "with-tags", "OPERATIONS": "8300000" }
     },
     "many-tags": {
-      "env": { "VARIANT": "many-tags", "ITERATIONS": "3000000" }
+      "env": { "VARIANT": "many-tags", "OPERATIONS": "3000000" }
     },
     "aggregated": {
-      "env": { "VARIANT": "aggregated", "ITERATIONS": "15000000" }
+      "env": { "VARIANT": "aggregated", "OPERATIONS": "15000000" }
     }
   }
 }

benchmark/sirun/encoding/index.js

@@ -10,7 +10,7 @@ const {
 } = process.env
 
 const WIDE_TAGS = Number(process.env.WIDE_TAGS) || 0
-const ENCODE_COUNT = Number(process.env.ENCODE_COUNT) || 5000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 const { AgentEncoder } = require(`../../../packages/dd-trace/src/encode/${ENCODER_VERSION}`)
 const { buildTrace, tickTrace, attachFreshEvents } = require('./trace-fixture')
@@ -43,12 +43,12 @@ encoder._reset()
 
 guard.loopStart()
 if (WITH_SPAN_EVENTS === 'none') {
-  for (let iteration = 0; iteration < ENCODE_COUNT; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     tickTrace(trace, iteration)
     encoder.encode(trace)
   }
 } else {
-  for (let iteration = 0; iteration < ENCODE_COUNT; iteration++) {
+  for (let iteration = 0; iteration < OPERATIONS; iteration++) {
     tickTrace(trace, iteration)
     attachFreshEvents(trace, iteration)
     encoder.encode(trace)

benchmark/sirun/encoding/meta.json

@@ -10,53 +10,53 @@
       "env": {
         "ENCODER_VERSION": "0.4",
         "WITH_SPAN_EVENTS": "none",
-        "ENCODE_COUNT": "31000"
+        "OPERATIONS": "31000"
       }
     },
     "0.5": {
       "env": {
         "ENCODER_VERSION": "0.5",
         "WITH_SPAN_EVENTS": "none",
-        "ENCODE_COUNT": "60000"
+        "OPERATIONS": "60000"
       }
     },
     "0.4-events-native": {
       "env": {
         "ENCODER_VERSION": "0.4",
         "WITH_SPAN_EVENTS": "native",
         "DD_TRACE_NATIVE_SPAN_EVENTS": "true",
-        "ENCODE_COUNT": "13000"
+        "OPERATIONS": "13000"
       }
     },
     "0.4-events-legacy": {
       "env": {
         "ENCODER_VERSION": "0.4",
         "WITH_SPAN_EVENTS": "legacy",
         "DD_TRACE_NATIVE_SPAN_EVENTS": "false",
-        "ENCODE_COUNT": "10500"
+        "OPERATIONS": "10500"
       }
     },
     "0.5-events-legacy": {
       "env": {
         "ENCODER_VERSION": "0.5",
         "WITH_SPAN_EVENTS": "legacy",
-        "ENCODE_COUNT": "15000"
+        "OPERATIONS": "15000"
       }
     },
     "0.4-wide-tags": {
       "env": {
         "ENCODER_VERSION": "0.4",
         "WITH_SPAN_EVENTS": "none",
         "WIDE_TAGS": "40",
-        "ENCODE_COUNT": "11000"
+        "OPERATIONS": "11000"
       }
     },
     "0.5-wide-tags": {
       "env": {
         "ENCODER_VERSION": "0.5",
         "WITH_SPAN_EVENTS": "none",
         "WIDE_TAGS": "40",
-        "ENCODE_COUNT": "19000"
+        "OPERATIONS": "19000"
       }
     }
   }

benchmark/sirun/exporting-pipeline/index.js

@@ -17,7 +17,7 @@ const id = require('../../../packages/dd-trace/src/id')
 // bench, and the real flush is a deferred unref'd timer that barely fires in a
 // short run. A no-op exporter keeps the loop CPU-bound, leaves memory flat (the
 // formatted chunk is discarded each pass) and drops the agent dependency.
-const COUNT = Number(process.env.COUNT) || 200_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 const WITH_STATS = process.env.WITH_STATS === '1'
 const WITH_LINKS = process.env.WITH_LINKS === '1'
 
@@ -101,7 +101,7 @@ if (WITH_LINKS) {
 
 guard.loopStart()
 exported = 0
-for (let i = 0; i < COUNT; i++) {
+for (let i = 0; i < OPERATIONS; i++) {
   // process() erases trace.finished each pass; restore the chunk so every
   // iteration formats the full set.
   trace.started = finished

benchmark/sirun/exporting-pipeline/meta.json

@@ -9,20 +9,20 @@
     "format": {
       "env": {
         "WITH_STATS": "0",
-        "COUNT": "200000"
+        "OPERATIONS": "200000"
       }
     },
     "format-with-stats": {
       "env": {
         "WITH_STATS": "1",
-        "COUNT": "200000"
+        "OPERATIONS": "200000"
       }
     },
     "format-with-links-events": {
       "env": {
         "WITH_STATS": "0",
         "WITH_LINKS": "1",
-        "COUNT": "85000"
+        "OPERATIONS": "85000"
       }
     }
   }

benchmark/sirun/fs/index.js

@@ -12,7 +12,7 @@ require('../../../packages/datadog-instrumentations/src/fs')
 const { channel } = require('../../../packages/datadog-instrumentations/src/helpers/instrument')
 
 const { VARIANT } = process.env
-const ITERATIONS = Number(process.env.ITERATIONS) || 6_000_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 const startChannel = channel('apm:fs:operation:start')
 const finishChannel = channel('apm:fs:operation:finish')
@@ -90,10 +90,10 @@ assert.deepEqual(
 
 guard.loopStart()
 let sink = 0
-for (let i = 0; i < ITERATIONS; i++) {
+for (let i = 0; i < OPERATIONS; i++) {
   sink += instrumentedCall()
 }
 guard.done()
 
 // orphan-guard variant returns 0 each call; subscribed returns 1.
-assert.equal(sink, VARIANT === 'subscribed' ? ITERATIONS : 0, 'unexpected sink')
+assert.equal(sink, VARIANT === 'subscribed' ? OPERATIONS : 0, 'unexpected sink')

benchmark/sirun/fs/meta.json

@@ -7,7 +7,7 @@
   "instructions": true,
   "variants": {
     "subscribed": {
-      "env": { "VARIANT": "subscribed", "ITERATIONS": "857000" }
+      "env": { "VARIANT": "subscribed", "OPERATIONS": "857000" }
     }
   }
 }

benchmark/sirun/iast/index.js

@@ -15,7 +15,7 @@ const ops = require('../../../packages/dd-trace/src/appsec/iast/taint-tracking/o
 //   sink-check - one transaction, taint once, then loop the isTainted/getRanges
 //     check that fires at every sink (the most frequent op)
 const { VARIANT } = process.env
-const ITERATIONS = Number(process.env.ITERATIONS) || 1_000_000
+const OPERATIONS = Number(process.env.OPERATIONS)
 
 const SOURCE = 'echo hello; cat /etc/passwd && rm -rf /tmp/x'
 const UNTAINTED = 'echo a static safe command with no user input'
@@ -31,7 +31,7 @@ if (VARIANT === 'sink-check') {
   assert.ok(ops.getRanges(iastContext, tainted).length > 0, 'tainted source has no ranges')
 
   guard.loopStart()
-  for (let i = 0; i < ITERATIONS; i++) {
+  for (let i = 0; i < OPERATIONS; i++) {
     if (ops.isTainted(iastContext, tainted)) {
       sink += ops.getRanges(iastContext, tainted).length
     }
@@ -50,7 +50,7 @@ if (VARIANT === 'sink-check') {
   ops.removeTransaction(probe)
 
   guard.loopStart()
-  for (let i = 0; i < ITERATIONS; i++) {
+  for (let i = 0; i < OPERATIONS; i++) {
     const iastContext = {}
     ops.createTransaction(String(i), iastContext)
     const tainted = ops.newTaintedString(iastContext, SOURCE, 'param', TYPE)

benchmark/sirun/iast/meta.json

@@ -7,10 +7,10 @@
   "instructions": true,
   "variants": {
     "request-lifecycle": {
-      "env": { "VARIANT": "request-lifecycle", "ITERATIONS": "8000" }
+      "env": { "VARIANT": "request-lifecycle", "OPERATIONS": "8000" }
     },
     "sink-check": {
-      "env": { "VARIANT": "sink-check", "ITERATIONS": "5000000" }
+      "env": { "VARIANT": "sink-check", "OPERATIONS": "5000000" }
     }
   }
 }