chore: test ddb rds with different flavors (#46)

Author: mosheshaham-dash0

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ name: CI
 on:
   pull_request:
     branches: [main, master]
+    types: [opened, synchronize, reopened, ready_for_review]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
@@ -19,6 +20,7 @@ env:
 jobs:
   fmt:
     name: Format
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -29,6 +31,7 @@ jobs:
 
   test:
     name: Test
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -38,6 +41,7 @@ jobs:
 
   python-distro-test:
     name: Python Distro Unit Tests
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -54,6 +58,7 @@ jobs:
 
   node-distro-test:
     name: Node Distro Unit Tests
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -71,6 +76,7 @@ jobs:
 
   build:
     name: Build Rust Binaries
+    if: github.event.pull_request.draft == false
     uses: ./.github/workflows/build.yml
 
   deploy:
@@ -106,6 +112,15 @@ jobs:
         with:
           role-to-assume: ${{ vars.AWS_ROLE_ARN }}
           aws-region: ${{ vars.AWS_REGION }}
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghcrunner
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf /usr/share/swift
+          docker system prune -af
+          df -h
       - name: Install dependencies
         working-directory: integration-tests/iac
         run: npm ci
@@ -114,6 +129,8 @@ jobs:
         run: npm run deploy
         env:
           DASH0_DEV_API_TOKEN: ${{ secrets.DASH0_DEV_API_TOKEN }}
+          TEST_POSTGRESS_PASSWORD: ${{ secrets.TEST_POSTGRESS_PASSWORD }}
+          TEST_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
           RESOURCE_PREFIX: ${{ env.RESOURCE_PREFIX }}
       - name: Install test dependencies
         working-directory: integration-tests/tests

.github/workflows/cleanup-pr.yml

@@ -33,6 +33,8 @@ jobs:
         run: npx cdk destroy ${RESOURCE_PREFIX}IntegrationTestsStack --force || true
         env:
           DASH0_DEV_API_TOKEN: ${{ secrets.DASH0_DEV_API_TOKEN }}
+          TEST_POSTGRESS_PASSWORD: ${{ secrets.TEST_POSTGRESS_PASSWORD }}
+          TEST_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
       - name: Delete PR-specific Lambda layers
         run: |
           for layer in ${RESOURCE_PREFIX}dash0-extension-python ${RESOURCE_PREFIX}dash0-extension-node ${RESOURCE_PREFIX}dash0-extension-java ${RESOURCE_PREFIX}dash0-extension-manual; do

README.md

@@ -2,11 +2,26 @@
 
 An extension for capturing observability data from AWS Lambda invocations and shipping to Dash0.
 
-This extension has four main functionalities:
+## Table of Contents
+
+- [Layer ARNs](#layer-arns)
+- [Configuration](#configuration)
+  - [Required](#required)
+  - [Optional](#optional)
+  - [Secret Masking](#secret-masking)
+- [Manual Instrumentation](#manual-instrumentation)
+- [Enrichment Attributes](#enrichment-attributes)
+  - [Span Attributes](#span-attributes)
+  - [Log Attributes](#log-attributes)
+  - [Metrics](#metrics)
+- [Dockerized Lambdas](#dockerized-lambdas)
+
+This extension has five main functionalities:
 1. Enable auto-instrumentation for supported runtimes, which currently include Python, Node, Java.
 2. Receive traces from auto/manual instrumentations, enrich with data acquired in the extension, and send to Dash0.
 3. Detect runtime errors such as timeout or out of memory and create synthetic traces for them
 4. Collect all logs and send to Dash0, correlated with the trace id of the invocation.
+5. Create metrics for invocation duration, cold start duration, billed duration, and memory used.
 
 
 ## Layer ARNs
@@ -42,8 +57,6 @@ See the release page for the latest ARNs of the extension layers for each runtim
 
 * `DASH0_REQUEST_TIMEOUT` - Timeout in milliseconds for HTTP requests to the backend. Default: `2000`.
 
-* `DASH0_MAX_EVENT_PAYLOAD` - Maximum size in KB for event payloads (request/response bodies) captured in traces. Payloads exceeding this limit are truncated. Default: `20`.
-
 * `DASH0_CREATE_PAYLOAD_LOG_RECORDS` - When set to `true` (the default), the extension creates log records containing the request and response payloads for the lambda invocation and each client call. Set to `false` to disable. Default: `true`.
 
 * `DASH0_DISABLE_TELEMETRY_LOG_COLLECTION` - When set to `true`, disables collecting logs from the [Lambda Telemetry API](https://docs.aws.amazon.com/lambda/latest/dg/telemetry-api.html). Default: `false`.
@@ -102,6 +115,16 @@ The following environment variables allow fine-grained control over secret maski
   Example: `DASH0_MASK_QUERY_PARAMS='[".*api_key.*", ".*token.*"]'`
 
 
+## Manual Instrumentation
+
+If you prefer to set up OpenTelemetry instrumentation yourself instead of relying on the extension's auto-instrumentation, you can use the manual layer and point your OTLP exporters to the extension's local endpoint. The extension will receive the telemetry, enrich it, and forward it to Dash0.
+
+1. Add the manual layer to your Lambda function: `arn:aws:lambda:<region>:115813213817:layer:dash0-extension-manual:<version>`.
+2. Configure your OTLP trace exporter to send to `http://127.0.0.1:9009/v1/traces`.
+3. If exporting metrics, configure your OTLP metric exporter to send to `http://127.0.0.1:9009/v1/metrics`.
+4. Make sure to flush all telemetry before the Lambda invocation completes (e.g., in a response hook or before returning the response).
+
+
 ## Enrichment Attributes
 
 The extension enriches telemetry data with additional attributes beyond what the auto-instrumentation provides.

integration-tests/iac/bin/integration-tests.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import * as cdk from 'aws-cdk-lib/core';
 import { IntegrationTestsStack } from '../lib/integration-tests-stack';
+import { SharedDbStack } from '../lib/shared-db-stack';
 
 const prefix = process.env.RESOURCE_PREFIX ?? '';
 const app = new cdk.App();
@@ -19,3 +20,5 @@ new IntegrationTestsStack(app, `${prefix}IntegrationTestsStack`, {
 
   /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
 });
+
+new SharedDbStack(app, 'SharedDbStack');

integration-tests/iac/lambdas/manual/tracing.js

@@ -18,9 +18,6 @@ const resource = resourceFromAttributes({
 
 const exporter = new OTLPTraceExporter({
   url: `http://127.0.0.1:9009/v1/traces`,
-  headers: {
-    Authorization: `Bearer ${process.env.DASH0_TOKEN}`,
-  },
 });
 
 const provider = new NodeTracerProvider({

integration-tests/iac/lambdas/node/rds_mysql.mjs

@@ -0,0 +1,44 @@
+import mysql from 'mysql2/promise';
+
+export async function handler(event, context) {
+    const connection = await mysql.createConnection({
+        host: process.env.DB_HOST,
+        port: parseInt(process.env.DB_PORT || '3306'),
+        database: process.env.DB_NAME,
+        user: process.env.DB_USER,
+        password: process.env.DB_PASSWORD,
+        ssl: { rejectUnauthorized: false },
+    });
+
+    try {
+        await connection.execute(
+            `CREATE TABLE IF NOT EXISTS test_entries (
+                id INT AUTO_INCREMENT PRIMARY KEY,
+                request_id TEXT NOT NULL,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        `);
+
+        const [insertResult] = await connection.execute(
+            'INSERT INTO test_entries (request_id) VALUES (?)',
+            [context.awsRequestId],
+        );
+
+        const [rows] = await connection.execute(
+            'SELECT * FROM test_entries WHERE request_id = ?',
+            [context.awsRequestId],
+        );
+
+        console.log(`Inserted and read back entry for request: ${context.awsRequestId}`);
+
+        return {
+            statusCode: 200,
+            body: JSON.stringify({
+                inserted: { id: insertResult.insertId, request_id: context.awsRequestId },
+                read: rows[0],
+            }),
+        };
+    } finally {
+        await connection.end();
+    }
+}

integration-tests/iac/lambdas/node/rds_postgres.mjs

@@ -0,0 +1,46 @@
+import { Client } from 'pg';
+
+export async function handler(event, context) {
+    const client = new Client({
+        host: process.env.DB_HOST,
+        port: parseInt(process.env.DB_PORT || '5432'),
+        database: process.env.DB_NAME,
+        user: process.env.DB_USER,
+        password: process.env.DB_PASSWORD,
+        ssl: { rejectUnauthorized: false },
+    });
+
+    await client.connect();
+
+    try {
+        await client.query(`
+            CREATE TABLE IF NOT EXISTS test_entries (
+                id SERIAL PRIMARY KEY,
+                request_id TEXT NOT NULL,
+                created_at TIMESTAMPTZ DEFAULT NOW()
+            )
+        `);
+
+        const insertResult = await client.query(
+            'INSERT INTO test_entries (request_id) VALUES ($1) RETURNING *',
+            [context.awsRequestId],
+        );
+
+        const readResult = await client.query(
+            'SELECT * FROM test_entries WHERE request_id = $1',
+            [context.awsRequestId],
+        );
+
+        console.log(`Inserted and read back entry for request: ${context.awsRequestId}`);
+
+        return {
+            statusCode: 200,
+            body: JSON.stringify({
+                inserted: insertResult.rows[0],
+                read: readResult.rows[0],
+            }),
+        };
+    } finally {
+        await client.end();
+    }
+}

integration-tests/iac/lambdas/python-db/rds_mysql.py

@@ -0,0 +1,53 @@
+import json
+import os
+import pymysql
+
+
+def handler(event, context):
+    conn = pymysql.connect(
+        host=os.environ["DB_HOST"],
+        port=int(os.environ.get("DB_PORT", "3306")),
+        database=os.environ["DB_NAME"],
+        user=os.environ["DB_USER"],
+        password=os.environ["DB_PASSWORD"],
+        ssl={"ssl": True},
+    )
+
+    try:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                CREATE TABLE IF NOT EXISTS test_entries (
+                    id INT AUTO_INCREMENT PRIMARY KEY,
+                    request_id TEXT NOT NULL,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+                """
+            )
+            conn.commit()
+
+            cur.execute(
+                "INSERT INTO test_entries (request_id) VALUES (%s)",
+                (context.aws_request_id,),
+            )
+            conn.commit()
+
+            cur.execute(
+                "SELECT * FROM test_entries WHERE request_id = %s",
+                (context.aws_request_id,),
+            )
+            read = cur.fetchone()
+
+        print(f"Inserted and read back entry for request: {context.aws_request_id}")
+
+        return {
+            "statusCode": 200,
+            "body": json.dumps(
+                {
+                    "inserted": {"request_id": context.aws_request_id},
+                    "read": str(read),
+                }
+            ),
+        }
+    finally:
+        conn.close()

integration-tests/iac/lambdas/python-db/rds_postgres.py

@@ -0,0 +1,54 @@
+import json
+import os
+import psycopg2
+
+
+def handler(event, context):
+    conn = psycopg2.connect(
+        host=os.environ["DB_HOST"],
+        port=int(os.environ.get("DB_PORT", "5432")),
+        dbname=os.environ["DB_NAME"],
+        user=os.environ["DB_USER"],
+        password=os.environ["DB_PASSWORD"],
+        sslmode="require",
+    )
+
+    try:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                CREATE TABLE IF NOT EXISTS test_entries (
+                    id SERIAL PRIMARY KEY,
+                    request_id TEXT NOT NULL,
+                    created_at TIMESTAMPTZ DEFAULT NOW()
+                )
+                """
+            )
+            conn.commit()
+
+            cur.execute(
+                "INSERT INTO test_entries (request_id) VALUES (%s) RETURNING *",
+                (context.aws_request_id,),
+            )
+            inserted = cur.fetchone()
+            conn.commit()
+
+            cur.execute(
+                "SELECT * FROM test_entries WHERE request_id = %s",
+                (context.aws_request_id,),
+            )
+            read = cur.fetchone()
+
+        print(f"Inserted and read back entry for request: {context.aws_request_id}")
+
+        return {
+            "statusCode": 200,
+            "body": json.dumps(
+                {
+                    "inserted": str(inserted),
+                    "read": str(read),
+                }
+            ),
+        }
+    finally:
+        conn.close()

integration-tests/iac/lambdas/python-db/requirements.txt

@@ -0,0 +1,2 @@
+psycopg2-binary
+pymysql