server.js

'use strict'

const tracer = require('dd-trace').init()
const { trace, ROOT_CONTEXT, SpanKind, propagation, metrics } = require('@opentelemetry/api')
tracer.use('express', false)
tracer.use('http', false)
tracer.use('dns', false)

const { MANUAL_KEEP, MANUAL_DROP } = require('dd-trace/ext')
const SpanContext = require('dd-trace/packages/dd-trace/src/opentracing/span_context')
const OtelSpanContext = require('dd-trace/packages/dd-trace/src/opentelemetry/span_context')

const { millisToHrTime } = require('@opentelemetry/core')

const { OpenFeature } = require('@openfeature/server-sdk')
let openFeatureClient = null

const { TracerProvider } = tracer
const tracerProvider = new TracerProvider()
tracerProvider.register()

const express = require('express');

const app = express();
app.use(express.json());

let dummyIdIncrementer = 10000000

// Map log level to OpenTelemetry severity
const SEVERITY_MAP = {
  'TRACE': { text: 'TRACE', number: 1 },
  'DEBUG': { text: 'DEBUG', number: 5 },
  'INFO': { text: 'INFO', number: 9 },
  'WARN': { text: 'WARN', number: 13 },
  'ERROR': { text: 'ERROR', number: 17 },
  'FATAL': { text: 'FATAL', number: 21 }
}

function microLongToHrTime (timestamp) {
  if (timestamp === null) {
      return [0, 0]
  }
  return [
      Math.floor(timestamp / 1000000),
      (timestamp % 1000000) * 1000,
  ]
}

const otelStatusCodes = {
  'UNSET': 0,
  'OK': 1,
  'ERROR': 2
}

const otelSpanKinds = {
  0: SpanKind.INTERNAL,
  1: SpanKind.SERVER,
  2: SpanKind.CLIENT,
  3: SpanKind.PRODUCER,
  4: SpanKind.CONSUMER
}

const spans = new Map()
const ddContext = new Map()
const otelSpans = new Map()
const otelMeters = new Map()
const otelMeterInstruments = new Map()
const loggerDict = {}

// Helper function to create instrument keys
function createInstrumentKey(meterName, name, kind, unit, description) {
  return `${meterName}:${name}:${kind}:${unit}:${description}`;
}

app.post('/trace/span/inject_headers', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  const http_headersDict = {};
  const http_headers = [];

  tracer.inject(span, 'http_headers', http_headersDict);
  for (const [key, value] of Object.entries(http_headersDict)) {
      http_headers.push([key, value]);
  }

  res.json({ http_headers });
});

app.post('/trace/span/extract_headers', (req, res) => {
  const request = req.body;
  const http_headers = request.http_headers || [];
  // Node.js HTTP headers are automatically lower-cased, simulate that here.
  const linkHeaders = Object.fromEntries(http_headers.map(([k, v]) => [k.toLowerCase(), v]));
  const extracted = tracer.extract('http_headers', linkHeaders);

  let extractedSpanID = null;
  const dummyTracer = require('dd-trace').init()
  const extractPropagator = dummyTracer._tracer._config.tracePropagationStyle.extract

  if (extractPropagator.includes('baggage') && extracted && !extracted._spanId && !extracted._traceId) {
    // baggage propagation does not require ids so http_headers could contain no ids
    // several endpoints in this file rely on having ids so we need to have dummy ids for internal use
    extracted._spanId = dummyIdIncrementer
    extracted._traceId = dummyIdIncrementer
    dummyIdIncrementer += 1
  }

  if (extracted && extracted._spanId) {
    extractedSpanID = extracted.toSpanId();
    ddContext[extractedSpanID] = extracted;
  }

  res.json({ span_id: extractedSpanID });
});

app.get('/trace/agent/ensure_agent_info', (req, res) => {
  res.json({ ready: true })
})

app.get('/trace/crash', (req, res) => {
  process.kill(process.pid, 'SIGSEGV');
  res.json({});
});

app.post('/trace/span/start', (req, res) => {
  const request = req.body;
  let parent = spans[request.parent_id] || ddContext[request.parent_id];

  // dd-trace's startSpan only special-cases `service` from the tags; `resource` and `type` are not
  // mapped (that mapping only happens in tracer.trace()). Set the canonical tag keys directly so the
  // span carries the requested resource and type. Only set them when provided so the resource keeps
  // defaulting to the operation name.
  const tags = {service: request.service};
  if (request.resource) tags['resource.name'] = request.resource;
  if (request.type) tags['span.type'] = request.type;
  for (const [key, value] of request.span_tags) tags[key] = value

  const span = tracer.startSpan(request.name, {
    childOf: parent,
    tags
  });

  if (ddContext[request.parent_id]) {
    for (const link of ddContext[request.parent_id]._links || []) span.addLink(link)
  }

  spans[span.context().toSpanId()] = span;
  res.json({ span_id: span.context().toSpanId(), trace_id:span.context().toTraceId() });
});

app.post('/trace/span/add_link', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  if (spans[request.parent_id]) {
    span.addLink({ context: spans[request.parent_id].context(), attributes: request.attributes })
  } else {
    span.addLink({ context: ddContext[request.parent_id], attributes: request.attributes })
  }
  res.json({});
});

app.post('/trace/span/finish', (req, res) => {
  const id = req.body.span_id
  const span = spans[id]
  span.finish()
  res.json({});
});

app.post('/trace/span/flush', (req, res) => {
  const { _tracer: { _exporter: { _writer } } } = tracer
  _writer.flush(() => {
    res.json({});
  })
  spans.clear();
  ddContext.clear();
});

app.post('/trace/span/set_meta', (req, res) => {
  const args = req.body;
  const spanId = args.span_id;
  const key = args.key;
  const value = args.value;
  const span = spans[spanId]
  span.setTag(key, value)
  res.json({});
});

app.post('/trace/span/set_metric', (req, res) => {
  const args = req.body;
  const spanId = args.span_id;
  const key = args.key;
  const value = args.value;
  const span = spans[spanId];
  span.setTag(key, value);
  res.json({});
});

app.post('/trace/span/manual_keep', (req, res) => {
  const args = req.body;
  const spanId = args.span_id;
  const span = spans[spanId];
  span.setTag(MANUAL_KEEP, true);
  res.json({});
});

app.post('/trace/span/manual_drop', (req, res) => {
  const args = req.body;
  const spanId = args.span_id;
  const span = spans[spanId];
  span.setTag(MANUAL_DROP, true);
  res.json({});
});

app.post('/trace/stats/flush', (req, res) => {
  // dd-trace-js implements CSS via SpanStatsProcessor on the SpanProcessor.
  // There is no public flush API, so reach into internals and wait for the
  // span-stats writer's HTTP send to complete before responding.
  const processor = tracer?._tracer?._processor?._stats
  if (!processor) {
    res.json({})
    return
  }
  const writer = processor.exporter?._writer
  if (!writer || typeof writer._sendPayload !== 'function') {
    processor.onInterval()
    res.json({})
    return
  }
  const originalSend = writer._sendPayload.bind(writer)
  let sendInvoked = false
  let responded = false
  const respond = () => {
    if (responded) return
    responded = true
    writer._sendPayload = originalSend
    res.json({})
  }
  writer._sendPayload = (data, count, done) => {
    sendInvoked = true
    originalSend(data, count, () => {
      done()
      respond()
    })
  }
  try {
    processor.onInterval()
  } catch (e) {
    respond()
    return
  }
  if (!sendInvoked) {
    respond()
  }
});

app.post('/trace/span/error', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  span.addTags({
      'error.msg': request.message,
      'error.type': request.type,
      'error.stack': request.stack
  })
  res.json({});
});

app.post('/trace/span/set_baggage', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  span.setBaggageItem(request.key, request.value)
  res.json({});
});

app.get('/trace/span/get_baggage', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  const baggage = span.getBaggageItem(request.key)
  res.json({ baggage });
});

app.get('/trace/span/get_all_baggage', (req, res) => {
const request = req.body;
  const span = spans[request.span_id]
  const baggage = span.getAllBaggageItems()
  res.json({ baggage: JSON.parse(baggage) });
});

app.post('/trace/span/remove_baggage', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  const baggages = span.removeBaggageItem(request.key)
  res.json({});
});

app.post('/trace/span/remove_all_baggage', (req, res) => {
  const request = req.body;
  const span = spans[request.span_id]
  const baggages = span.removeAllBaggageItems()
  res.json({});
});

app.post('/trace/otel/start_span', (req, res) => {
  const request = req.body;
  const otelTracer = tracerProvider.getTracer()

  const makeSpan = (parentContext) => {

    const links = (request.links || []).map(link => {
      let spanContext = otelSpans[link.parent_id].spanContext()
      return {context: spanContext, attributes: link.attributes}
    });

    let kind = null
    if (request.kind != null) {
      kind = request.kind - 1
    }
    const span = otelTracer.startSpan(request.name, {
        type: request.type,
        kind: otelSpanKinds[request.span_kind],
        attributes: request.attributes,
        links,
        startTime: microLongToHrTime(request.timestamp)
    }, parentContext)
    const ctx = span._ddSpan.context()
    const span_id = "0x" + ctx._spanId.toString(16)
    const trace_id = ctx._traceId.toString(10)

    otelSpans[span_id] = span
    res.json({ span_id, trace_id });
  }
  if (request.parent_id && request.parent_id !== 0) {
      const parentSpan = otelSpans[request.parent_id]
      const parentContext = trace.setSpan(ROOT_CONTEXT, parentSpan)
      return makeSpan(parentContext)
  }

  makeSpan()
});

app.post('/trace/otel/end_span', (req, res) => {
  const { id, timestamp } = req.body;
  const span_id = `${id}`
  const span = otelSpans[span_id]
  span.end(microLongToHrTime(timestamp))
  res.json({});
});

app.post('/trace/otel/flush', async (req, res) => {
  tracerProvider.forceFlush().then(function() {
    otelSpans.clear()
    res.json({ success: true })
  })
  .catch(function(rej) {
    console.log(rej)
    res.json({ success: false })
  });
});

app.post('/trace/otel/is_recording', (req, res) => {
  const { span_id } = req.body;
  const span = otelSpans[span_id]
  res.json({ is_recording: span.isRecording() });
});

app.post('/trace/otel/span_context', (req, res) => {
  const { span_id } = req.body;
  const span = otelSpans[span_id]
  const ctx = span.spanContext()
  res.json({
    span_id: `0x${ctx.spanId}`,
    trace_id: ctx.traceId,
    // Node.js official OTel API uses a number, not a string
    trace_flags: `0${ctx.traceFlags}`,
    trace_state: ctx.traceState.serialize(),

    // TODO: What is this and where is it supposed to come from? 🤔
    remote: ctx.is_remote?ctx.is_remote:false,
  });
});

app.post('/trace/otel/set_status', (req, res) => {
  const { span_id, code, description } = req.body;
  const span = otelSpans[span_id]
  span.setStatus({
      code: otelStatusCodes[code],
      message: description
  })
  res.json({});
});

app.post('/trace/otel/set_name', (req, res) => {
  const { span_id, name } = req.body;
  const span = otelSpans[span_id]
  span.updateName(name)
  res.json({});
});

app.post('/trace/otel/set_attributes', (req, res) => {
  const { span_id, attributes } = req.body;
  const span = otelSpans[span_id]
  span.setAttributes(attributes)
  res.json({});
});

app.get('/trace/config', (req, res) => {
  // The tracer's resolved config is an internal shape whose property paths rename across
  // refactors, so this endpoint reports nothing from it. Node config consistency is asserted
  // via telemetry and observable behaviour in the parametric suite; the test client fills
  // every documented key with null, so the cross-language parity contract still holds.
  res.json({ config: {} })
});

app.post("/otel/logger/create", (req, res) => {
  const { name, level, version, schema_url, attributes } = req.body

  if (loggerDict[name]) {
    return res.status(200).json({ success: false })
  }

  const { logs } = require('@opentelemetry/api-logs')
  const loggerProvider = logs.getLoggerProvider()

  const loggerOptions = schema_url ? { schemaUrl: schema_url } : {}
  const loggerVersion = version || ''
  const logger = loggerProvider.getLogger(name, loggerVersion, loggerOptions)

  loggerDict[name] = logger
  res.status(200).json({ success: true })
})

app.post("/otel/logger/write", (req, res) => {
  const { logs } = require('@opentelemetry/api-logs')
  const { logger_name, level, message, span_id } = req.body

  const logger = loggerDict[logger_name]
  if (!logger) {
    return res.status(400).json({ error: `Logger ${logger_name} not found` })
  }

  let otelContext = undefined
  const otelSpan = otelSpans[span_id]
  const ddSpan = spans[span_id]
  if (otelSpan) {
    otelContext = trace.setSpan(ROOT_CONTEXT, otelSpan)
  } else if (ddSpan) {
    const ddSpanContext = ddSpan.context()
    otelContext = trace.setSpanContext(ROOT_CONTEXT, {
      spanId: ddSpanContext.toSpanId(true),
      traceId: ddSpanContext.toTraceId(true),
      flags: ddSpanContext._sampling.priority >= 0 ? 1 : 0
    })
  }

  const severity = SEVERITY_MAP[level.toUpperCase()]
  logger.emit( {
    severityText: severity.text,
    severityNumber: severity.number,
    body: message,
    context: otelContext
  })
  res.status(200).json({ success: true })
})

app.post("/log/otel/flush", (req, res) => {
  const { logs } = require('@opentelemetry/api-logs')

  try {
    // Get the current logs provider
    const logsProvider = logs.getLoggerProvider()
    const providerType = logsProvider.constructor.name

    // Force flush all logs with timeout
    const timeoutMs = (req.body.seconds || 5) * 1000
    logsProvider.forceFlush(timeoutMs)
      .then(() => {
        res.status(200).json({ success: true, message: providerType })
      })
      .catch((error) => {
        res.status(200).json({ success: false, message: `Error: ${error.message}` })
      })
  } catch (error) {
    res.status(200).json({ success: false, message: `Error: ${error.message}` })
  }
})

app.post("/trace/otel/add_event", (req, res) => {
  const { span_id, name, timestamp, attributes } = req.body;
  const span = otelSpans[span_id]
  // convert to TimeInput object using millisToHrTime
  span.addEvent(name, attributes, millisToHrTime(timestamp / 1000))
  res.json({})
})

app.post("/trace/otel/record_exception", (req, res) => {
  const { span_id, message, attributes } = req.body;
  const span = otelSpans[span_id]
  span.recordException(new Error(message))
  res.json({})
})

app.post("/trace/otel/otel_set_baggage", (req, res) => {
  const bag = propagation
        .createBaggage()
        .setEntry(req.body.key, { value: req.body.value });
  const context = propagation.setBaggage(ROOT_CONTEXT, bag)
  const value = propagation.getBaggage(context).getEntry(req.body.key).value
  res.json({ value });
});

// Feature Flag & Experimentation endpoints
app.post('/ffe/start', async (req, res) => {
  const hasFeatureFlaggingConfiguration = [
    'DD_FEATURE_FLAGS_ENABLED',
    'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE',
    'DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL',
    'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'
  ].some(name => process.env[name] !== undefined)

  if (hasFeatureFlaggingConfiguration) {
    try {
      const { openfeature } = tracer
      await OpenFeature.setProviderAndWait(openfeature)
    } catch {
      openFeatureClient = OpenFeature.getClient()
      res.status(500).json({})
      return
    }
  }
  openFeatureClient = OpenFeature.getClient()
  res.json({})
})

app.post('/ffe/evaluate', async (req, res) => {
  const { flag, variationType, defaultValue, targetingKey, attributes, span_id } = req.body;
  let value, reason;
  const context = { targetingKey, ...attributes }

  // Helper function to perform the actual flag evaluation
  const doEvaluate = async () => {
    try {
      switch (variationType) {
        case 'BOOLEAN':
          value = await openFeatureClient.getBooleanValue(flag, defaultValue, context)
          break;
        case 'STRING':
          value = await openFeatureClient.getStringValue(flag, defaultValue, context)
          break;
        case 'INTEGER':
          value = await openFeatureClient.getNumberValue(flag, defaultValue, context)
          break;
        case 'NUMERIC':
          value = await openFeatureClient.getNumberValue(flag, defaultValue, context)
          break;
        case 'JSON':
          value = await openFeatureClient.getObjectValue(flag, defaultValue, context)
          break;
        default:
          value = defaultValue;
      }

      reason = 'DEFAULT';
    } catch (error) {
      value = defaultValue;
      reason = 'ERROR';
    }
  }

  // If a span_id is provided, activate that span during the evaluation
  // so that the SpanEnrichmentHook can find the root span
  if (span_id && spans[span_id]) {
    await tracer.scope().activate(spans[span_id], doEvaluate)
  } else {
    await doEvaluate()
  }

  res.json({
    value: value,
    reason: reason
  });
});

// OpenTelemetry Metrics Endpoints

app.post('/metrics/otel/get_meter', (req, res) => {
  const { name, version, schema_url, attributes } = req.body;
  if (!otelMeters.has(name)) {
    const meterProvider = metrics.getMeterProvider();
    const meter = meterProvider.getMeter(name, version, { schemaUrl: schema_url, attributes });
    otelMeters.set(name, meter);
  }
  res.json({});
});

app.post('/metrics/otel/create_counter', (req, res) => {
  const { meter_name, name, description, unit } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const counter = meter.createCounter(name, { unit, description });

  const instrumentKey = createInstrumentKey(meter_name, name, 'counter', unit, description);
  otelMeterInstruments.set(instrumentKey, counter);
  res.json({});
});

app.post('/metrics/otel/counter_add', (req, res) => {
  const { meter_name, name, unit, description, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const instrumentKey = createInstrumentKey(meter_name, name, 'counter', unit, description);
  if (!otelMeterInstruments.has(instrumentKey)) {
    return res.status(400).json({
      error: `Instrument with identifying fields Name=${name},Kind=Counter,Unit=${unit},Description=${description} not found in registered instruments for Meter=${meter_name}`
    });
  }

  const counter = otelMeterInstruments.get(instrumentKey);
  counter.add(value, attributes);
  res.json({});
});

app.post('/metrics/otel/create_updowncounter', (req, res) => {
  const { meter_name, name, description, unit } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const counter = meter.createUpDownCounter(name, { unit, description });

  const instrumentKey = createInstrumentKey(meter_name, name, 'updowncounter', unit, description);
  otelMeterInstruments.set(instrumentKey, counter);
  res.json({});
});

app.post('/metrics/otel/updowncounter_add', (req, res) => {
  const { meter_name, name, unit, description, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const instrumentKey = createInstrumentKey(meter_name, name, 'updowncounter', unit, description);
  if (!otelMeterInstruments.has(instrumentKey)) {
    return res.status(400).json({
      error: `Instrument with identifying fields Name=${name},Kind=UpDownCounter,Unit=${unit},Description=${description} not found in registered instruments for Meter=${meter_name}`
    });
  }

  const counter = otelMeterInstruments.get(instrumentKey);
  counter.add(value, attributes);
  res.json({});
});

app.post('/metrics/otel/create_gauge', (req, res) => {
  const { meter_name, name, description, unit } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const gauge = meter.createGauge(name, { unit, description });

  const instrumentKey = createInstrumentKey(meter_name, name, 'gauge', unit, description);
  otelMeterInstruments.set(instrumentKey, gauge);
  res.json({});
});

app.post('/metrics/otel/gauge_record', (req, res) => {
  const { meter_name, name, unit, description, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const instrumentKey = createInstrumentKey(meter_name, name, 'gauge', unit, description);
  if (!otelMeterInstruments.has(instrumentKey)) {
    return res.status(400).json({
      error: `Instrument with identifying fields Name=${name},Kind=Gauge,Unit=${unit},Description=${description} not found in registered instruments for Meter=${meter_name}`
    });
  }

  const gauge = otelMeterInstruments.get(instrumentKey);
  gauge.record(value, attributes);
  res.json({});
});

app.post('/metrics/otel/create_histogram', (req, res) => {
  const { meter_name, name, description, unit } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const histogram = meter.createHistogram(name, { unit, description });

  const instrumentKey = createInstrumentKey(meter_name, name, 'histogram', unit, description);
  otelMeterInstruments.set(instrumentKey, histogram);
  res.json({});
});

app.post('/metrics/otel/histogram_record', (req, res) => {
  const { meter_name, name, unit, description, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const instrumentKey = createInstrumentKey(meter_name, name, 'histogram', unit, description);
  if (!otelMeterInstruments.has(instrumentKey)) {
    return res.status(400).json({
      error: `Instrument with identifying fields Name=${name},Kind=Histogram,Unit=${unit},Description=${description} not found in registered instruments for Meter=${meter_name}`
    });
  }

  const histogram = otelMeterInstruments.get(instrumentKey);
  histogram.record(value, attributes);
  res.json({});
});

app.post('/metrics/otel/create_asynchronous_counter', (req, res) => {
  const { meter_name, name, description, unit, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const observableCounter = meter.createObservableCounter(name, {
    unit,
    description
  });

  // Add the callback
  observableCounter.addCallback((observableResult) => {
    observableResult.observe(value, attributes);
  });

  const instrumentKey = createInstrumentKey(meter_name, name, 'observable_counter', unit, description);
  otelMeterInstruments.set(instrumentKey, observableCounter);
  res.json({});
});

app.post('/metrics/otel/create_asynchronous_updowncounter', (req, res) => {
  const { meter_name, name, description, unit, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const observableUpDownCounter = meter.createObservableUpDownCounter(name, {
    unit,
    description
  });

  // Add the callback
  observableUpDownCounter.addCallback((observableResult) => {
    observableResult.observe(value, attributes);
  });

  const instrumentKey = createInstrumentKey(meter_name, name, 'observable_updowncounter', unit, description);
  otelMeterInstruments.set(instrumentKey, observableUpDownCounter);
  res.json({});
});

app.post('/metrics/otel/create_asynchronous_gauge', (req, res) => {
  const { meter_name, name, description, unit, value, attributes } = req.body;
  if (!otelMeters.has(meter_name)) {
    return res.status(400).json({
      error: `Meter name ${meter_name} not found in registered meters`
    });
  }

  const meter = otelMeters.get(meter_name);
  const observableGauge = meter.createObservableGauge(name, {
    unit,
    description
  });

  // Add the callback
  observableGauge.addCallback((observableResult) => {
    observableResult.observe(value, attributes);
  });

  const instrumentKey = createInstrumentKey(meter_name, name, 'observable_gauge', unit, description);
  otelMeterInstruments.set(instrumentKey, observableGauge);
  res.json({});
});

app.post('/metrics/otel/force_flush', (req, res) => {
  const meterProvider = metrics.getMeterProvider();
  if (meterProvider.reader) {
    meterProvider.reader.forceFlush()
    res.json({ success: true });
  } else {
    res.json({ success: false, message: 'Force flush not supported' });
  }
});

// add LLM Observability routes
const addLlmObsRoutes = require('./llmobs');
addLlmObsRoutes(app);

const port = process.env.APM_TEST_CLIENT_SERVER_PORT;
app.listen(port, () => {
  console.log(`Server is running on port ${port}`);
});