Lambda observability: Opentelemetry & XRay traces

config/.env.example

@@ -0,0 +1,7 @@
+METASTORE_CONFIG=config/metastore.yaml
+JWT_SECRET=secret
+RUST_LOG=info
+TRACING_LEVEL=info
+OTEL_SERVICE_NAME=embucket-lambda-api
+HONEYCOMB_API_KEY=full_ingest_key
+HONEYCOMB_ENDPOINT_URL=api.honeycomb.io:443

crates/api-snowflake-rest/src/tests/create_test_server.rs

@@ -30,6 +30,7 @@ pub fn executor_default_cfg() -> UtilsConfig {
     UtilsConfig::default().with_max_concurrency_level(2)
 }
 
+#[must_use]
 pub fn metastore_default_settings_cfg() -> MetastoreSettingsConfig {
     MetastoreSettingsConfig::default()
         .with_object_store_connect_timeout(1)

crates/embucket-lambda/Cargo.toml

@@ -12,8 +12,6 @@ executor = { path = "../executor" }
 build-info = { path = "../build-info" }
 lambda_http = "0.17"
 tokio = { workspace = true }
-tracing = { workspace = true }
-tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
 base64 = "0.22"
 serde = { workspace = true }
 serde_json = { workspace = true }
@@ -26,6 +24,13 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"]
 tower = { workspace = true }
 tower-http = { workspace = true }
 cfg-if = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { version = "0.3.20", features = ["env-filter", "registry", "fmt", "json"] }
+tracing-opentelemetry = { version = "0.31.0" }
+tracing-allocations = { version = "0.1.0", optional = true }
+opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic"] }
+opentelemetry_sdk = { version = "0.30.0" }
+opentelemetry = { version = "0.30.0" }
 
 [lints]
 workspace = true
@@ -54,7 +59,3 @@ tracing = "Active"
 # Note: include path is relative to workspace root
 # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap
 include = ["config"]
-
-[package.metadata.lambda.deploy.env]
-LOG_FORMAT = "json"
-METASTORE_CONFIG = "config/metastore.yaml"

crates/embucket-lambda/Makefile

@@ -2,20 +2,47 @@
 
 # Function name (override with: make deploy FUNCTION_NAME=your-function)
 FUNCTION_NAME ?= embucket-lambda
-ENV_FILE ?= .env
+
+ENV_FILE ?= config/.env
+IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole
+# --layer-arn required for flexibiity
+# otel-collector-config goes second as it overrides original config  from opentelemetry-collector 
+OTEL_COLLECTOR_LAYERS ?= \
+	--layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1\
+	--layer-arn arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:22\
+	--layer-arn arn:aws:lambda:us-east-2:580247275435:layer:LambdaInsightsExtension-Arm64:33
+
 # supported features: "streaming"
 FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES))
 
 build:
-	cd ../.. && cargo lambda build --release --arm64 --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM)
+	cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --include config/metastore.yaml --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM)
 
 # Deploy to AWS (must run from workspace root for include paths to work)
 deploy: build deploy-only
 	@echo "Deployed $(FUNCTION_NAME) to AWS"
 
 # Quick deploy without rebuild
 deploy-only:
-	cd ../.. && cargo lambda deploy --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE)
+	@test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy <function-name>" >&2; exit 1)
+	cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME)
+	aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true
+	@$(MAKE) url FUNCTION_NAME=$(FUNCTION_NAME)
+
+url:
+	@test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make url <function-name>" >&2; exit 1)
+	@set -e; \
+	echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \
+	aws lambda create-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null 2>&1 || \
+	  aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \
+	echo "Ensuring public invoke permission exists..." ; \
+	aws lambda add-permission --function-name "$(FUNCTION_NAME)" \
+	  --statement-id AllowPublicURLInvoke \
+	  --action lambda:InvokeFunctionUrl \
+	  --principal "*" \
+	  --function-url-auth-type NONE >/dev/null 2>&1 || true ; \
+	URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \
+	echo "$$URL"
 
 # Watch locally for development
 watch:
@@ -27,6 +54,7 @@ test:
 
 # Tail lambda logs
 logs:
+	@test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy <function-name>" >&2; exit 1)
 	aws logs tail /aws/lambda/$(FUNCTION_NAME) --since 5m --follow
 
 # Verify deployment with snow CLI

crates/embucket-lambda/README.md

@@ -32,6 +32,9 @@ make verify
 make logs
 ```
 
+`make deploy` will build, deploy with .env vars and with the iam-role in the makefile, add logs group, create a url, add allow uri access policy. And volia - it works. 
+P.S. Don't forget to change host url in the `snowcli` config.
+
 The function name defaults to `embucket-lambda` but can be overridden:
 - Via Makefile variable: `make deploy FUNCTION_NAME=my-function`
 - Via environment variable: `export FUNCTION_NAME=my-function && make deploy`
@@ -77,6 +80,21 @@ cargo lambda deploy --binary-name bootstrap
   ```
 - It will deploy envs from `.env` if `ENV_FILE` not specified
 
+### Observability
+
+
+#### AWS traces
+We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces.
+- `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE".
+
+#### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/)
+- Required environment variables configuring remote Observability platform:
+  * `HONEYCOMB_API_KEY`
+  * `HONEYCOMB_ENDPOINT_URL`
+- Optional:
+  * `OTEL_SERVICE_NAME`
+  - `TRACING_LEVEL` - verbosity level, default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE".
+
 ### Test locally
 
 ```bash
@@ -107,5 +125,6 @@ aws logs tail /aws/lambda/embucket-lambda --since 5m --follow
 - `LOG_FORMAT`: json
 - `METASTORE_CONFIG`: config/metastore.yaml
 - `RUST_LOG`: (optional) Set logging level, defaults to "info"
+- `TRACING_LEVEL`: (optional) Set tracing level, defaults to "info"
 
 

crates/embucket-lambda/extensions/collector-config/README.md

@@ -0,0 +1,9 @@
+To update the layer config:
+1. `cd crates/embucket-lambda/extensions`
+2. `zip -r otel-collector-config-layer.zip collector-config`
+3. `cd ../../..`
+4. `aws lambda publish-layer-version
+--layer-name otel-collector-config  
+--zip-file fileb://config/layer-root/otel-collector-config-layer.zip  
+--compatible-runtimes provided.al2 provided.al2023   
+--compatible-architectures arm64`

crates/embucket-lambda/extensions/collector-config/config.yml

@@ -0,0 +1,23 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  otlp:
+    endpoint: "${env:HONEYCOMB_ENDPOINT_URL}"
+    headers:
+      x-honeycomb-team: "${env:HONEYCOMB_API_KEY}"
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp]

crates/embucket-lambda/src/config.rs

@@ -25,6 +25,10 @@ pub struct EnvConfig {
     pub iceberg_catalog_timeout_secs: u64,
     pub object_store_timeout_secs: u64,
     pub object_store_connect_timeout_secs: u64,
+    pub otel_grpc: bool,
+    pub tracing_level: String,
+    pub log_format: String,
+    pub log_filter: String,
 }
 
 impl EnvConfig {
@@ -58,6 +62,10 @@ impl EnvConfig {
             object_store_timeout_secs: parse_env("OBJECT_STORE_TIMEOUT_SECS").unwrap_or(30),
             object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS")
                 .unwrap_or(3),
+            otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true),
+            tracing_level: env_or_default("TRACING_LEVEL", "INFO"),
+            log_format: env_or_default("LOG_FORMAT", "json"),
+            log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()),
         }
     }