feat: create otel metrics for lambda (#39)

Author: mosheshaham-dash0

README.md

@@ -98,6 +98,82 @@ The following environment variables allow fine-grained control over secret maski
   Example: `DASH0_MASK_QUERY_PARAMS='[".*api_key.*", ".*token.*"]'`
 
 
+## Enrichment Attributes
+
+The extension enriches telemetry data with additional attributes beyond what the auto-instrumentation provides.
+
+### Span Attributes
+
+The following attributes are added to spans by the extension (if relevant):
+
+| Attribute | Type | Description |
+|---|---|---|
+| `faas.invocation_id` | string | The AWS request ID of the current invocation. |
+| `faas.trigger` | string | The event source that triggered the Lambda (e.g., `aws:sqs`, `aws:dynamodb`, `aws:event_bridge`). Extracted from the event payload. |
+| `faas.init_duration` | double | The cold start initialization duration in milliseconds. Only present on cold start invocations. |
+| `dash0.faas.record_count` | int | The number of records in a batch event (SQS, DynamoDB Streams, Kinesis, SNS). |
+| `dash0.faas.trigger_arn` | string | The ARN of the event source (e.g., SQS queue ARN, DynamoDB stream ARN, SNS topic ARN). |
+| `dash0.faas.event_bridge_source` | string | The `source` field from an EventBridge event. |
+| `dash0.faas.event_bridge_detail_type` | string | The `detail-type` field from an EventBridge event. |
+
+#### Resource Attributes (Spans)
+
+These attributes are added to the resource of span data:
+
+| Attribute | Type | Description |
+|---|---|---|
+| `service.name` | string | The service name, from `OTEL_SERVICE_NAME` or defaults to `unknown_service`. |
+| `process.environment_variable.<KEY>` | string | Lambda environment variables (with sensitive values masked). Added to the span resource. |
+
+### Log Attributes
+
+The following attributes are added to log records by the extension (if relevant):
+
+| Attribute | Type | Description |
+|---|---|---|
+| `faas.invocation_id` | string | The AWS request ID, used to correlate logs with the invocation span. |
+| `dash0.faas.payload_type` | string | The type of payload log record. Values: `lambda_event`, `lambda_return_value`, `http_request_body`, `http_response_body`. Only present on payload log records. |
+
+#### Resource Attributes (Logs)
+
+These attributes are added to the resource of log data:
+
+| Attribute | Type | Description |
+|---|---|---|
+| `cloud.platform` | string | Always set to `aws_lambda`. |
+| `cloud.resource.id` | string | The full ARN of the Lambda function. |
+| `cloud.account.id` | string | The AWS account ID. |
+| `service.name` | string | The service name, from `OTEL_SERVICE_NAME` or defaults to `unknown_service`. |
+
+### Metrics
+
+The extension creates the following histogram metrics for each Lambda invocation:
+
+| Metric | Unit | Description |
+|---|---|---|
+| `faas.duration` | ms | Duration of the invocation. |
+| `faas.init_duration` | ms | Duration of the cold start initialization. Only present on cold start invocations. |
+| `dash0.faas.billed_duration` | ms | Billed duration of the invocation. |
+| `dash0.faas.memory_used` | MB | Memory used by the invocation. |
+
+#### Metric Attributes
+
+The following attributes are added to each metric data point:
+
+| Attribute | Type | Description |
+|---|---|---|
+| `cloud.resource_id` | string | The full ARN of the Lambda function. |
+| `cloud.account.id` | string | The AWS account ID. |
+
+#### Resource Attributes (Metrics)
+
+These attributes are added to the resource of metric data:
+
+| Attribute | Type | Description |
+|---|---|---|
+| `service.name` | string | The service name, from `OTEL_SERVICE_NAME` or defaults to `unknown_service`. |
+
+
 ## Dockerized Lambdas
 
 For containerized Lambda functions, use the provided Docker images in a multi-stage build. The extension images are available for Node.js, Python, and Java runtimes.

integration-tests/tests/src/test-java-exception.test.ts

@@ -2,6 +2,7 @@ import { describe, expect } from 'vitest';
 import {
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     getAttributesMap,
     LogToCheck,
@@ -61,6 +62,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambda invocation', () => {

integration-tests/tests/src/test-java-success.test.ts

@@ -2,6 +2,7 @@ import { describe } from 'vitest';
 import {
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     LogToCheck,
     invokeFunction,
@@ -47,6 +48,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambdainvocation', () => {

integration-tests/tests/src/test-node-exception.test.ts

@@ -2,6 +2,7 @@ import { describe, expect } from 'vitest';
 import {
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     getAttributesMap,
     LogToCheck,
@@ -60,6 +61,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambda invocation', () => {

integration-tests/tests/src/test-node-success.test.ts

@@ -3,6 +3,7 @@ import {
     checkHttpSpan,
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     getAttributesMap,
     LogToCheck,
@@ -72,6 +73,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambda invocation', () => {

integration-tests/tests/src/test-python-exception.test.ts

@@ -3,6 +3,7 @@ import {
     checkHttpSpan,
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     getAttributesMap,
     LogToCheck,
@@ -76,6 +77,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambda invocation', () => {

integration-tests/tests/src/test-python-success.test.ts

@@ -3,6 +3,7 @@ import {
     checkHttpSpan,
     checkLogs,
     checkMainSpans,
+    checkMetrics,
     checkOverheadSpan,
     getAttributesMap,
     LogToCheck,
@@ -69,6 +70,11 @@ const verifySuccessInvocation = async (functionName: string, invocationEnd: bool
         traceId,
         rootSpanId,
     });
+
+    await checkMetrics({
+        functionName,
+        metricNames: ['faas.duration', 'dash0.faas.billed_duration', 'dash0.faas.memory_used', 'faas.init_duration'],
+    });
 }
 
 describe.concurrent('Lambdainvocation', () => {

integration-tests/tests/src/utils.ts

@@ -454,6 +454,62 @@ export const checkOverheadSpan = async ({
     }
 }
 
+export const checkMetrics = async ({
+    functionName,
+    metricNames,
+}: {
+    functionName: string;
+    metricNames: string[];
+}): Promise<void> => {
+    for (const metricName of metricNames) {
+        let found = false;
+        for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+            await delay(RETRY_DELAY_MS);
+            console.log(`Attempt ${attempt} to fetch metric ${metricName} for ${functionName}`);
+            try {
+                const params = new URLSearchParams({
+                    dataset: DASH0_LAMBDA_TESTS_DATASET,
+                    start: 'now-10m',
+                    end: 'now',
+                    step: '1m',
+                    query: `{otel_metric_name = "${metricName}", otel_metric_type = "histogram", service_name = "${functionName}"}`,
+                });
+                const response = await fetch(DASH0_ENDPOINT + 'prometheus/api/v1/query_range', {
+                    method: 'POST',
+                    headers: {
+                        authorization: `Bearer ${DASH0_TOKEN}`,
+                        'content-type': 'application/x-www-form-urlencoded',
+                    },
+                    body: params.toString(),
+                });
+
+                const payload = await response.json() as any;
+                expect(payload.status).toEqual('success');
+
+                const results = payload.data?.result ?? [];
+                const hasValue = results.some((r: any) =>
+                    (r.values ?? []).some((v: any) => parseFloat(v[1]) >= 1)
+                );
+
+                if (hasValue) {
+                    found = true;
+                    break;
+                }
+
+                if (attempt === MAX_ATTEMPTS) {
+                    throw new Error(`Metric ${metricName} for ${functionName} not found with value >= 1`);
+                }
+            } catch (error) {
+                console.error(`Error fetching metric ${metricName} on attempt ${attempt}:`, error);
+                if (attempt === MAX_ATTEMPTS) {
+                    throw error;
+                }
+            }
+        }
+        expect(found, `Metric ${metricName} should have at least one value >= 1`).toBe(true);
+    }
+}
+
 export const checkException = (span: any, exception_type: string) => {
     const events = span.events;
     expect(events.length).toEqual(1);

src/extension/telemetry_receiver.rs

@@ -1,5 +1,6 @@
 use crate::config::user::is_telemetry_log_collection_disabled;
 use crate::otlp::exporter::{flush_telemetry_logs, send_traces};
+use crate::otlp::metrics_creation::create_supplementary_metrics;
 use crate::otlp::span_creation::{create_overhead_supplementary_span, create_supplementary_spans};
 use crate::otlp::span_mutations::build_synthetic_trace;
 use crate::state::invocation_entry;
@@ -44,6 +45,7 @@ pub async fn telemetry(req: Request<Body>) -> Result<Response<Body>, Error> {
             if log.r#type == "platform.report" {
                 if let Some(id) = &log.invocation_id {
                     create_overhead_supplementary_span(id);
+                    create_supplementary_metrics(id);
                     invocation_entry::update(id, |entry| {
                         entry.state = crate::state::invocation_entry::InvocationState::Done;
                         entry.init_duration = 0.0;

src/otlp/attributes.rs

@@ -0,0 +1,26 @@
+// OpenTelemetry semantic convention attribute keys
+pub const CLOUD_RESOURCE_ID: &str = "cloud.resource_id";
+pub const CLOUD_ACCOUNT_ID: &str = "cloud.account.id";
+pub const CLOUD_PLATFORM: &str = "cloud.platform";
+pub const CLOUD_RESOURCE_ID_SEMCONV: &str = "cloud.resource.id";
+pub const SERVICE_NAME: &str = "service.name";
+pub const FAAS_INVOCATION_ID: &str = "faas.invocation_id";
+pub const FAAS_TRIGGER: &str = "faas.trigger";
+pub const FAAS_INIT_DURATION: &str = "faas.init_duration";
+
+// Exception attributes
+pub const EXCEPTION_TYPE: &str = "exception.type";
+pub const EXCEPTION_MESSAGE: &str = "exception.message";
+pub const EXCEPTION_ESCAPED: &str = "exception.escaped";
+pub const EXCEPTION_STACKTRACE: &str = "exception.stacktrace";
+
+// HTTP attributes
+pub const HTTP_REQUEST_BODY: &str = "http.request.body";
+pub const HTTP_RESPONSE_BODY: &str = "http.response.body";
+
+// Dash0-specific attributes
+pub const DASH0_FAAS_RECORD_COUNT: &str = "dash0.faas.record_count";
+pub const DASH0_FAAS_TRIGGER_ARN: &str = "dash0.faas.trigger_arn";
+pub const DASH0_FAAS_EVENT_BRIDGE_SOURCE: &str = "dash0.faas.event_bridge_source";
+pub const DASH0_FAAS_EVENT_BRIDGE_DETAIL_TYPE: &str = "dash0.faas.event_bridge_detail_type";
+pub const DASH0_FAAS_PAYLOAD_TYPE: &str = "dash0.faas.payload_type";