From 4ca99d055a315b09eadb965f9d74401e8d570d82 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:12:37 +0100 Subject: [PATCH 01/31] add otel_grpc param --- crates/embucketd/src/cli.rs | 8 ++++++++ crates/embucketd/src/main.rs | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/embucketd/src/cli.rs b/crates/embucketd/src/cli.rs index f6db5875..6c8b3b83 100644 --- a/crates/embucketd/src/cli.rs +++ b/crates/embucketd/src/cli.rs @@ -118,6 +118,14 @@ pub struct CliOpts { )] pub auth_demo_password: Option, + #[arg( + long, + env = "OTEL_GRPC", + default_value = "true", + help = "Enable OTLP gRPC exporter (requires 'otel-grpc' feature)" + )] + pub otel_grpc: bool, + #[arg( long, value_enum, diff --git a/crates/embucketd/src/main.rs b/crates/embucketd/src/main.rs index 7f315aea..20aea838 100644 --- a/crates/embucketd/src/main.rs +++ b/crates/embucketd/src/main.rs @@ -207,11 +207,19 @@ async fn async_main( #[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] fn setup_tracing(opts: &cli::CliOpts) -> SdkTracerProvider { - // Initialize OTLP exporter using gRPC (Tonic) - let exporter = opentelemetry_otlp::SpanExporter::builder() - .with_tonic() - .build() - .expect("Failed to create OTLP exporter"); + let exporter = if opts.otel_grpc { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } else { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + }; let resource = Resource::builder().with_service_name("Em").build(); From f3743d950c4f63a256f696c53ee544209427e41a Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:13:29 +0100 Subject: [PATCH 02/31] use registry for both logs and spans --- Cargo.lock | 5 ++ crates/embucket-lambda/Cargo.toml | 9 ++- crates/embucket-lambda/src/config.rs | 8 +++ crates/embucket-lambda/src/main.rs | 94 ++++++++++++++++++++++++++-- 4 files changed, 109 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c56b98a..9e06c379 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3314,6 +3314,9 @@ dependencies = [ "http 1.4.0", "http-body-util", "lambda_http", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "serde", "serde_json", "serde_urlencoded", @@ -3321,6 +3324,8 @@ dependencies = [ "tower 0.5.2", "tower-http", "tracing", + "tracing-allocations", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index e03e8192..516023d7 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -12,8 +12,6 @@ executor = { path = "../executor" } build-info = { path = "../build-info" } lambda_http = "0.17" tokio = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } base64 = "0.22" serde = { workspace = true } serde_json = { workspace = true } @@ -26,6 +24,13 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"] tower = { workspace = true } tower-http = { workspace = true } cfg-if = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { version = "0.3.20", features = ["env-filter", "registry", "fmt", "json"] } +tracing-opentelemetry = { version = "0.31.0" } +tracing-allocations = { version = "0.1.0", optional = true } +opentelemetry-otlp = { version = "0.30.0", features = ["grpc-tonic"] } +opentelemetry_sdk = { version = "0.30.0" } +opentelemetry = { version = "0.30.0" } [lints] workspace = true diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index 800eb54b..a89d02af 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -25,6 +25,10 @@ pub struct EnvConfig { pub iceberg_catalog_timeout_secs: u64, pub object_store_timeout_secs: u64, pub object_store_connect_timeout_secs: u64, + pub otel_grpc: bool, + pub tracing_level: String, + pub log_format: String, + pub log_filter: String, } impl EnvConfig { @@ -58,6 +62,10 @@ impl EnvConfig { object_store_timeout_secs: parse_env("OBJECT_STORE_TIMEOUT_SECS").unwrap_or(30), object_store_connect_timeout_secs: parse_env("OBJECT_STORE_CONNECT_TIMEOUT_SECS") .unwrap_or(3), + otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), + tracing_level: env_or_default("TRACING_LEVEL", "INFO"), + log_format: env_or_default("LOG_FORMAT", "pretty"), + log_filter: parse_env("RUST_LOG").unwrap_or("INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 8fe894d2..7558020e 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -15,11 +15,20 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; +use tracing::instrument::WithSubscriber; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SdkTracerProvider; +use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { use lambda_http::run_with_streaming_response as run; @@ -30,6 +39,8 @@ cfg_if::cfg_if! { type InitResult = Result>; +const DISABLED_TARGETS: [&str; 2] = ["h2", "aws_smithy_runtime"]; + #[tokio::main] async fn main() -> Result<(), LambdaError> { init_tracing(); @@ -44,6 +55,10 @@ async fn main() -> Result<(), LambdaError> { ); let env_config = EnvConfig::from_env(); + + init_tracing(&env_config); + init_tracing_and_logs(&env_config); + info!( data_format = %env_config.data_format, max_concurrency = env_config.max_concurrency_level, @@ -230,14 +245,83 @@ fn extract_socket_addr(headers: &HeaderMap) -> Option { .map(|ip| SocketAddr::new(ip, 0)) } -fn init_tracing() { +#[allow(clippy::expect_used, clippy::redundant_closure_for_method_calls)] +fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { + let exporter = if config.otel_grpc { + // Initialize OTLP exporter using gRPC (Tonic) + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .build() + .expect("Failed to create OTLP gRPC exporter") + } else { + // Initialize OTLP exporter using HTTP + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP HTTP exporter") + }; + + let resource = Resource::builder().build(); + + let tracing_provider = SdkTracerProvider::builder() + .with_span_processor(BatchSpanProcessor::builder(exporter).build()) + .with_resource(resource) + .build(); + + let targets_with_level = + |targets: &[&'static str], level: LevelFilter| -> Vec<(&str, LevelFilter)> { + // let default_log_targets: Vec<(String, LevelFilter)> = + targets.iter().map(|t| ((*t), level)).collect() + }; + + + let registry = tracing_subscriber::registry() + // Telemetry filtering + .with( + tracing_opentelemetry::OpenTelemetryLayer::new(tracing_provider.tracer("embucket")) + .with_level(true) + .with_filter( + Targets::default() + .with_targets(targets_with_level(&DISABLED_TARGETS, LevelFilter::OFF)) + .with_default(config.tracing_level.parse().unwrap_or(tracing::Level::INFO)), + ), + ); + // Logs filtering + // fmt::layer has different types for json vs plain + if config.log_format == "json" { + registry.with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(false) + .json() + .with_current_span(true) + .with_span_list(true) + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); + } else { + registry.with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(std::io::stdout().is_terminal()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::ENTER + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ) + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); + }; + + tracing_provider +} + + +fn init_tracing(config: &EnvConfig) { let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); let emit_ansi = std::io::stdout().is_terminal(); - // Use json format if requested via env var, otherwise use pretty format with span events - let format = std::env::var("LOG_FORMAT").unwrap_or_else(|_| "pretty".to_string()); - - if format == "json" { + if config.log_format == "json" { let _ = tracing_subscriber::fmt() .with_env_filter(filter) .with_target(true) From f41c269b11f8060b3f0cb1a91dc4783986c00c83 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Sat, 13 Dec 2025 05:57:42 +0100 Subject: [PATCH 03/31] add collector artifacts into gitignore --- .gitignore | 2 ++ crates/embucket-lambda/Cargo.toml | 2 +- crates/embucket-lambda/src/main.rs | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6372e0a5..f537c353 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ debug/ target/ data/ +collector-config/ +extensions/ .env metastore.yaml diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index 516023d7..e70dbca4 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -58,7 +58,7 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config"] +include = ["config", "collector-config", "extensions"] [package.metadata.lambda.deploy.env] LOG_FORMAT = "json" diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 7558020e..efc00aa0 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -15,7 +15,6 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; -use tracing::instrument::WithSubscriber; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; @@ -25,9 +24,8 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::BatchSpanProcessor; use opentelemetry_sdk::trace::SdkTracerProvider; -use tracing_subscriber::filter::{FilterExt, LevelFilter, Targets, filter_fn}; +use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::EnvFilter; -use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { From bfbcb92f44d77dde06c7e482ebde31852ee0eebc Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Mon, 15 Dec 2025 13:08:55 +0100 Subject: [PATCH 04/31] shitdown tracing provider --- crates/embucket-lambda/src/config.rs | 2 +- crates/embucket-lambda/src/main.rs | 81 ++++++++++++++-------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index a89d02af..8058919b 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -65,7 +65,7 @@ impl EnvConfig { otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), log_format: env_or_default("LOG_FORMAT", "pretty"), - log_filter: parse_env("RUST_LOG").unwrap_or("INFO".to_string()), + log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index efc00aa0..69c7c1cb 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -15,17 +15,17 @@ use catalog_metastore::metastore_settings_config::MetastoreSettingsConfig; use http::HeaderMap; use http_body_util::BodyExt; use lambda_http::{Body as LambdaBody, Error as LambdaError, Request, Response, service_fn}; +use opentelemetry::trace::TracerProvider; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; -use opentelemetry::trace::TracerProvider; -use opentelemetry_sdk::Resource; -use opentelemetry_sdk::trace::BatchSpanProcessor; -use opentelemetry_sdk::trace::SdkTracerProvider; -use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::{LevelFilter, Targets}; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; cfg_if::cfg_if! { if #[cfg(feature = "streaming")] { @@ -41,21 +41,9 @@ const DISABLED_TARGETS: [&str; 2] = ["h2", "aws_smithy_runtime"]; #[tokio::main] async fn main() -> Result<(), LambdaError> { - init_tracing(); - - // Log version and build information on startup - info!( - version = %BuildInfo::GIT_DESCRIBE, - git_sha = %BuildInfo::GIT_SHA_SHORT, - git_branch = %BuildInfo::GIT_BRANCH, - build_timestamp = %BuildInfo::BUILD_TIMESTAMP, - "embucket-lambda started" - ); - let env_config = EnvConfig::from_env(); - init_tracing(&env_config); - init_tracing_and_logs(&env_config); + let tracing_provider = init_tracing_and_logs(&env_config); info!( data_format = %env_config.data_format, @@ -78,11 +66,18 @@ async fn main() -> Result<(), LambdaError> { err })?); - run(service_fn(move |event: Request| { + let err = run(service_fn(move |event: Request| { let app = Arc::clone(&app); async move { app.handle_event(event).await } })) - .await + .await; + + tracing_provider.shutdown().map_err(|err| { + error!(error = %err, "Failed to shutdown TracerProvider"); + err + })?; + + err } struct LambdaApp { @@ -287,29 +282,31 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Logs filtering // fmt::layer has different types for json vs plain if config.log_format == "json" { - registry.with( - tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(false) - .json() - .with_current_span(true) - .with_span_list(true) - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); + registry + .with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(false) + .json() + .with_current_span(true) + .with_span_list(true), + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); } else { - registry.with( - tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(std::io::stdout().is_terminal()) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::ENTER - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - ) - .with(EnvFilter::new(config.log_filter.clone())) - .init(); - }; + registry + .with( + tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(std::io::stdout().is_terminal()) + .with_span_events( + tracing_subscriber::fmt::format::FmtSpan::ENTER + | tracing_subscriber::fmt::format::FmtSpan::CLOSE, + ), + ) + .with(EnvFilter::new(config.log_filter.clone())) + .init(); + } tracing_provider } From ac6ba62b2243d93543b041d7848a45abf1f2e565 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Mon, 15 Dec 2025 21:37:40 +0200 Subject: [PATCH 05/31] fixes for honeycomb --- crates/embucket-lambda/src/config.rs | 2 +- crates/embucket-lambda/src/main.rs | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/embucket-lambda/src/config.rs b/crates/embucket-lambda/src/config.rs index 8058919b..90d4194f 100644 --- a/crates/embucket-lambda/src/config.rs +++ b/crates/embucket-lambda/src/config.rs @@ -64,7 +64,7 @@ impl EnvConfig { .unwrap_or(3), otel_grpc: parse_env("OTEL_GRPC").unwrap_or(true), tracing_level: env_or_default("TRACING_LEVEL", "INFO"), - log_format: env_or_default("LOG_FORMAT", "pretty"), + log_format: env_or_default("LOG_FORMAT", "json"), log_filter: parse_env("RUST_LOG").unwrap_or_else(|| "INFO".to_string()), } } diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 69c7c1cb..ac7f5e57 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -22,6 +22,7 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; +use opentelemetry_otlp::WithExportConfig; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -244,17 +245,19 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Initialize OTLP exporter using gRPC (Tonic) opentelemetry_otlp::SpanExporter::builder() .with_tonic() + .with_endpoint("http://127.0.0.1:4317".to_string()) .build() .expect("Failed to create OTLP gRPC exporter") } else { // Initialize OTLP exporter using HTTP opentelemetry_otlp::SpanExporter::builder() .with_http() + .with_endpoint("http://127.0.0.1:4318".to_string()) .build() .expect("Failed to create OTLP HTTP exporter") }; - let resource = Resource::builder().build(); + let resource = Resource::builder().with_service_name("embucket-lambda-api").build(); let tracing_provider = SdkTracerProvider::builder() .with_span_processor(BatchSpanProcessor::builder(exporter).build()) @@ -285,11 +288,12 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { registry .with( tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(false) .json() - .with_current_span(true) - .with_span_list(true), + .with_target(false) + .with_ansi(false) + .with_current_span(false) + .with_span_list(false) + .without_time(), ) .with(EnvFilter::new(config.log_filter.clone())) .init(); From 7c7e62308c3302d7e2584d8765cd927ff02a7c1a Mon Sep 17 00:00:00 2001 From: DanCodedThis <94703934+DanCodedThis@users.noreply.github.com> Date: Mon, 15 Dec 2025 23:52:25 +0200 Subject: [PATCH 06/31] tracing: OTLP layers (#91) * fixes for honeycomb and otlp layers * fixes for honeycomb and otlp layers * fmt fix * fix * fix * hardcoded * not hardcoded * remove duplication * remove duplication --- .../api-snowflake-rest/src/tests/create_test_server.rs | 1 + crates/embucket-lambda/Makefile | 5 +++-- crates/embucket-lambda/src/main.rs | 9 +-------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/crates/api-snowflake-rest/src/tests/create_test_server.rs b/crates/api-snowflake-rest/src/tests/create_test_server.rs index ebd0f535..96b3a1be 100644 --- a/crates/api-snowflake-rest/src/tests/create_test_server.rs +++ b/crates/api-snowflake-rest/src/tests/create_test_server.rs @@ -30,6 +30,7 @@ pub fn executor_default_cfg() -> UtilsConfig { UtilsConfig::default().with_max_concurrency_level(2) } +#[must_use] pub fn metastore_default_settings_cfg() -> MetastoreSettingsConfig { MetastoreSettingsConfig::default() .with_object_store_connect_timeout(1) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index d60d679b..cfdc6b37 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -11,11 +11,12 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only - @echo "Deployed $(FUNCTION_NAME) to AWS" + @echo "Deployed $(FUNCTION_NAME) to AWS" # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE) + cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 + --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE) # Watch locally for development watch: diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index ac7f5e57..e23ef01c 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -22,7 +22,6 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; -use opentelemetry_otlp::WithExportConfig; use tower::ServiceExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -163,10 +162,6 @@ impl LambdaApp { ); } - // if let Err(err) = ensure_session_header(&mut parts.headers, &self.state).await { - // return Ok(snowflake_error_response(&err)); - // } - let mut axum_request = to_axum_request(parts, body_bytes); if let Some(addr) = extract_socket_addr(axum_request.headers()) { axum_request.extensions_mut().insert(ConnectInfo(addr)); @@ -245,19 +240,17 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { // Initialize OTLP exporter using gRPC (Tonic) opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .with_endpoint("http://127.0.0.1:4317".to_string()) .build() .expect("Failed to create OTLP gRPC exporter") } else { // Initialize OTLP exporter using HTTP opentelemetry_otlp::SpanExporter::builder() .with_http() - .with_endpoint("http://127.0.0.1:4318".to_string()) .build() .expect("Failed to create OTLP HTTP exporter") }; - let resource = Resource::builder().with_service_name("embucket-lambda-api").build(); + let resource = Resource::builder().build(); let tracing_provider = SdkTracerProvider::builder() .with_span_processor(BatchSpanProcessor::builder(exporter).build()) From cb71b11f551fb50488607f8358f080daef8b2ba9 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Tue, 16 Dec 2025 09:59:26 +0100 Subject: [PATCH 07/31] staged --- crates/embucket-lambda/Cargo.toml | 6 +----- crates/embucket-lambda/Makefile | 8 +++++--- crates/embucket-lambda/README.md | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index e70dbca4..83ebe5a3 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -58,8 +58,4 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config", "collector-config", "extensions"] - -[package.metadata.lambda.deploy.env] -LOG_FORMAT = "json" -METASTORE_CONFIG = "config/metastore.yaml" +include = ["config", "collector-config", "extensions"] \ No newline at end of file diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index cfdc6b37..12f35e6d 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -3,6 +3,9 @@ # Function name (override with: make deploy FUNCTION_NAME=your-function) FUNCTION_NAME ?= embucket-lambda ENV_FILE ?= .env +IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole +# --layer-arn required for flexibiity +OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) @@ -11,12 +14,11 @@ build: # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only - @echo "Deployed $(FUNCTION_NAME) to AWS" + @echo "Deployed $(FUNCTION_NAME) to AWS" # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1 arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 - --binary-name bootstrap $(FUNCTION_NAME) --env-file $(ENV_FILE) + cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development watch: diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index 9cb44dbb..c82dbb6d 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -77,6 +77,22 @@ cargo lambda deploy --binary-name bootstrap ``` - It will deploy envs from `.env` if `ENV_FILE` not specified +### Observability + + +#### AWS traces +We send events, spans to stdout log in json format, and in case if AWS X-Ray is enabled it enhances traces. +- `RUST_LOG` - Controls verbosity log level. Default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". + +#### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/) +- Required environment variables configuring remote Observability platform: + * `HONEYCOMB_API_KEY` + * `HONEYCOMB_DATASET` + * `ENDPOINT_URL` (should become `HONEYCOMB_ENDPOINT_URL`) +- Optional: + * `OTEL_SERVICE_NAME` + - `TRACING_LEVEL` - verbosity level, default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". + ### Test locally ```bash @@ -107,5 +123,6 @@ aws logs tail /aws/lambda/embucket-lambda --since 5m --follow - `LOG_FORMAT`: json - `METASTORE_CONFIG`: config/metastore.yaml - `RUST_LOG`: (optional) Set logging level, defaults to "info" +- `TRACING_LEVEL`: (optional) Set tracing level, defaults to "info" From ccc4f449a891ad838a01f95800968d07f6bb4210 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 14:24:36 +0200 Subject: [PATCH 08/31] fix make build --- crates/embucket-lambda/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 12f35e6d..4ec7090d 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -1,16 +1,16 @@ .PHONY: build deploy test logs clean # Function name (override with: make deploy FUNCTION_NAME=your-function) -FUNCTION_NAME ?= embucket-lambda -ENV_FILE ?= .env +FUNCTION_NAME ?= +ENV_FILE ?= embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 +OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) build: - cd ../.. && cargo lambda build --release --arm64 --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) + cd ../.. && cargo lambda build --release -p embucket-lambda --arm64 -o zip --include config/metastore.yaml --manifest-path crates/embucket-lambda/Cargo.toml $(FEATURES_PARAM) # Deploy to AWS (must run from workspace root for include paths to work) deploy: build deploy-only @@ -18,7 +18,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: - cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development watch: From 29019261321dd1d75f9f00c9dc97ca5caeb66be1 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 14:37:50 +0200 Subject: [PATCH 09/31] fix make build --- crates/embucket-lambda/Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 4ec7090d..09392257 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -2,6 +2,16 @@ # Function name (override with: make deploy FUNCTION_NAME=your-function) FUNCTION_NAME ?= + +# Allow: make deploy my-function (without writing FUNCTION_NAME=...) +EXTRA_GOALS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) +ifneq ($(strip $(EXTRA_GOALS)),) + FUNCTION_NAME := $(word 1,$(EXTRA_GOALS)) +endif + +# Swallow the extra goal(s) so make doesn't treat them as real targets +$(eval $(EXTRA_GOALS):;@:) + ENV_FILE ?= embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity @@ -18,6 +28,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: + @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) # Watch locally for development @@ -30,6 +41,7 @@ test: # Tail lambda logs logs: + @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) aws logs tail /aws/lambda/$(FUNCTION_NAME) --since 5m --follow # Verify deployment with snow CLI From 00194c8ca955db633e16a14d5c87ee113ed0a497 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 15:54:05 +0200 Subject: [PATCH 10/31] fix make build --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 09392257..b0d55116 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -15,7 +15,7 @@ $(eval $(EXTRA_GOALS):;@:) ENV_FILE ?= embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:19 +OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:20 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) From 49420b244d7f64476fce05dbc0a14ca3224d8239 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 15:57:46 +0200 Subject: [PATCH 11/31] fix make build --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index b0d55116..63d4b92b 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -12,7 +12,7 @@ endif # Swallow the extra goal(s) so make doesn't treat them as real targets $(eval $(EXTRA_GOALS):;@:) -ENV_FILE ?= embucket-lambda/.env +ENV_FILE ?= crates/embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:20 From 49d289ba0db7f5f9c2a7c5b497f2add5d7eb2cb6 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:03:06 +0200 Subject: [PATCH 12/31] fix make delpoy logs group auto creation --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 63d4b92b..f9abf7c8 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -29,7 +29,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) - cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) && aws logs create-log-group --log-group-name /aws/lambda/$(FUNCTION_NAME) # Watch locally for development watch: From a57e8802ed1df944160d80128e0ebc6ecdf89d6d Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:03:53 +0200 Subject: [PATCH 13/31] fix make delpoy logs group auto creation --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index f9abf7c8..92309e6f 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -29,7 +29,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) - cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) && aws logs create-log-group --log-group-name /aws/lambda/$(FUNCTION_NAME) + cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) && aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" # Watch locally for development watch: From b8dc78a59a0a49c383e2260962da8deeda60d78e Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:12:51 +0200 Subject: [PATCH 14/31] autoamticl url and hhtp access rule --- crates/embucket-lambda/Makefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 92309e6f..05f05475 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -30,6 +30,22 @@ deploy: build deploy-only deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) && aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" + @$(MAKE) url FUNCTION_NAME=$(FUNCTION_NAME) + +url: + @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make url " >&2; exit 1) + @set -e; \ + echo "Ensuring Function URL config exists for $(FUNCTION_NAME) (auth: NONE)..." ; \ + aws lambda create-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null 2>&1 || \ + aws lambda update-function-url-config --function-name "$(FUNCTION_NAME)" --auth-type NONE >/dev/null ; \ + echo "Ensuring public invoke permission exists..." ; \ + aws lambda add-permission --function-name "$(FUNCTION_NAME)" \ + --statement-id AllowPublicURLInvoke \ + --action lambda:InvokeFunctionUrl \ + --principal "*" \ + --function-url-auth-type NONE >/dev/null 2>&1 || true ; \ + URL="$$(aws lambda get-function-url-config --function-name "$(FUNCTION_NAME)" --query 'FunctionUrl' --output text)"; \ + echo "$$URL" # Watch locally for development watch: From 21ab90c09c5d10608dba3380eec2e313020db5ef Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:15:57 +0200 Subject: [PATCH 15/31] fix final hopefully --- crates/embucket-lambda/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 05f05475..19492e03 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -29,7 +29,8 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) - cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) && aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" + cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" @$(MAKE) url FUNCTION_NAME=$(FUNCTION_NAME) url: From 2d327d99e4f35c961fc429f1a85e8bb7347fa968 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:39:38 +0200 Subject: [PATCH 16/31] deploy via cargo command --- .cargo/config.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..ef40dc8d --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[alias] +lambda-deploy = "!sh -c 'make -C crates/embucket-lambda deploy \"$1\"' --" +lambda-deploy-only = "!sh -c 'make -C crates/embucket-lambda deploy-only \"$1\"' --" \ No newline at end of file From 218774023650f4675a54eb55e240a89d2c0323da Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 16:44:32 +0200 Subject: [PATCH 17/31] env exmaple --- .cargo/config.toml | 3 --- crates/embucket-lambda/.env.example | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) delete mode 100644 .cargo/config.toml create mode 100644 crates/embucket-lambda/.env.example diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index ef40dc8d..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,3 +0,0 @@ -[alias] -lambda-deploy = "!sh -c 'make -C crates/embucket-lambda deploy \"$1\"' --" -lambda-deploy-only = "!sh -c 'make -C crates/embucket-lambda deploy-only \"$1\"' --" \ No newline at end of file diff --git a/crates/embucket-lambda/.env.example b/crates/embucket-lambda/.env.example new file mode 100644 index 00000000..2e24fb00 --- /dev/null +++ b/crates/embucket-lambda/.env.example @@ -0,0 +1,7 @@ +METASTORE_CONFIG=config/metastore.yaml +JWT_SECRET=secret +RUST_LOG=info +LOG_FORMAT=json +OTEL_SERVICE_NAME=embucket-lambda-api +HONEYCOMB_API_KEY=full_ingest_key +ENDPOINT_URL=api.honeycomb.io:443 \ No newline at end of file From 4d2b0af1167c3f9fd77b3b1781770069f309dce8 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 17:02:56 +0200 Subject: [PATCH 18/31] small readme --- crates/embucket-lambda/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index c82dbb6d..bc2145f9 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -20,7 +20,7 @@ cd crates/embucket-lambda make deploy # Deploy to a different function -make deploy FUNCTION_NAME=my-other-function +make deploy my-other-function # Deploy without rebuilding make deploy-only @@ -32,6 +32,9 @@ make verify make logs ``` +`make deploy` will build, deploy with .env vars and with the iam-role in the makefile, add logs group, create a url, add allow uri access policy. And volia - it works. +P.S. Don't forget to change host url in the `snowcli` config. + The function name defaults to `embucket-lambda` but can be overridden: - Via Makefile variable: `make deploy FUNCTION_NAME=my-function` - Via environment variable: `export FUNCTION_NAME=my-function && make deploy` From c53e5e8ba14a68dd27684b1d7756f7c9db165136 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 17:17:23 +0200 Subject: [PATCH 19/31] fmt --- crates/embucket-lambda/src/main.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index e23ef01c..8360bcb1 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -263,7 +263,6 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { targets.iter().map(|t| ((*t), level)).collect() }; - let registry = tracing_subscriber::registry() // Telemetry filtering .with( @@ -307,30 +306,3 @@ fn init_tracing_and_logs(config: &EnvConfig) -> SdkTracerProvider { tracing_provider } - - -fn init_tracing(config: &EnvConfig) { - let filter = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); - let emit_ansi = std::io::stdout().is_terminal(); - // Use json format if requested via env var, otherwise use pretty format with span events - if config.log_format == "json" { - let _ = tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(true) - .with_ansi(false) - .json() - .with_current_span(true) - .with_span_list(true) - .try_init(); - } else { - let _ = tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(true) - .with_ansi(emit_ansi) - .with_span_events( - tracing_subscriber::fmt::format::FmtSpan::ENTER - | tracing_subscriber::fmt::format::FmtSpan::CLOSE, - ) - .try_init(); - } -} From 4c2822e53433e87e6853203270e0fd7825ebd9ba Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 17:32:46 +0200 Subject: [PATCH 20/31] fixes --- crates/embucket-lambda/.env.example | 2 +- crates/embucket-lambda/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/embucket-lambda/.env.example b/crates/embucket-lambda/.env.example index 2e24fb00..fa9b9770 100644 --- a/crates/embucket-lambda/.env.example +++ b/crates/embucket-lambda/.env.example @@ -4,4 +4,4 @@ RUST_LOG=info LOG_FORMAT=json OTEL_SERVICE_NAME=embucket-lambda-api HONEYCOMB_API_KEY=full_ingest_key -ENDPOINT_URL=api.honeycomb.io:443 \ No newline at end of file +HONEYCOMB_ENDPOINT_URL=api.honeycomb.io:443 \ No newline at end of file diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 19492e03..71c75ba2 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -15,7 +15,7 @@ $(eval $(EXTRA_GOALS):;@:) ENV_FILE ?= crates/embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:20 +OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:21 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) From 479920100cbb53971e058944450cb6fe2301193f Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 17:46:53 +0200 Subject: [PATCH 21/31] fixes --- .gitignore | 1 - config/layer-root/collector-config/README.md | 13 +++++++++++ config/layer-root/collector-config/config.yml | 23 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 config/layer-root/collector-config/README.md create mode 100644 config/layer-root/collector-config/config.yml diff --git a/.gitignore b/.gitignore index f537c353..67a52c5f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ debug/ target/ data/ -collector-config/ extensions/ .env diff --git a/config/layer-root/collector-config/README.md b/config/layer-root/collector-config/README.md new file mode 100644 index 00000000..a0f27f8e --- /dev/null +++ b/config/layer-root/collector-config/README.md @@ -0,0 +1,13 @@ +To update the layer config: +1. `cd config/layer-root` +2. `zip -r otel-collector-config-layer.zip collector-config` +3. `cd ../..` +4. `aws lambda publish-layer-version +--layer-name otel-collector-config +--zip-file fileb://config/layer-root/otel-collector-config-layer.zip +--compatible-runtimes provided.al2 provided.al2023 +--compatible-architectures arm64` + + + + diff --git a/config/layer-root/collector-config/config.yml b/config/layer-root/collector-config/config.yml new file mode 100644 index 00000000..958d0fb6 --- /dev/null +++ b/config/layer-root/collector-config/config.yml @@ -0,0 +1,23 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + otlp: + endpoint: "${env:HONEYCOMB_ENDPOINT_URL}" + headers: + x-honeycomb-team: "${env:HONEYCOMB_API_KEY}" + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] \ No newline at end of file From d7ae9754f36afc20810cce80569de1d0334c408a Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Tue, 16 Dec 2025 17:01:33 +0100 Subject: [PATCH 22/31] remove experiment files --- crates/embucket-lambda/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Cargo.toml b/crates/embucket-lambda/Cargo.toml index 83ebe5a3..b99aa8bb 100644 --- a/crates/embucket-lambda/Cargo.toml +++ b/crates/embucket-lambda/Cargo.toml @@ -58,4 +58,4 @@ timeout = 30 tracing = "Active" # Note: include path is relative to workspace root # Must run deploy from workspace root: cargo lambda deploy --binary-name bootstrap -include = ["config", "collector-config", "extensions"] \ No newline at end of file +include = ["config"] From c6b601bd68db47e5361592744608ae76029628e1 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Tue, 16 Dec 2025 17:18:36 +0100 Subject: [PATCH 23/31] fixes --- .gitignore | 1 - crates/embucket-lambda/.env.example | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 67a52c5f..6372e0a5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ debug/ target/ data/ -extensions/ .env metastore.yaml diff --git a/crates/embucket-lambda/.env.example b/crates/embucket-lambda/.env.example index fa9b9770..e456ae9c 100644 --- a/crates/embucket-lambda/.env.example +++ b/crates/embucket-lambda/.env.example @@ -1,7 +1,7 @@ METASTORE_CONFIG=config/metastore.yaml JWT_SECRET=secret RUST_LOG=info -LOG_FORMAT=json +TRACING_LEVEL=info OTEL_SERVICE_NAME=embucket-lambda-api HONEYCOMB_API_KEY=full_ingest_key HONEYCOMB_ENDPOINT_URL=api.honeycomb.io:443 \ No newline at end of file From 16f6f46cb835587ced490b361696948856234f94 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 21:15:34 +0200 Subject: [PATCH 24/31] fixes --- crates/embucket-lambda/Makefile | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 71c75ba2..576bb4d8 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -1,21 +1,12 @@ .PHONY: build deploy test logs clean # Function name (override with: make deploy FUNCTION_NAME=your-function) -FUNCTION_NAME ?= - -# Allow: make deploy my-function (without writing FUNCTION_NAME=...) -EXTRA_GOALS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) -ifneq ($(strip $(EXTRA_GOALS)),) - FUNCTION_NAME := $(word 1,$(EXTRA_GOALS)) -endif - -# Swallow the extra goal(s) so make doesn't treat them as real targets -$(eval $(EXTRA_GOALS):;@:) +FUNCTION_NAME ?= embucket-lambda ENV_FILE ?= crates/embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:21 +OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:21 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) @@ -29,8 +20,8 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) - cd ../.. && cargo lambda deploy --layer-arn $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) - aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" + cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true @$(MAKE) url FUNCTION_NAME=$(FUNCTION_NAME) url: From 3d3d7a92c699b6c3f5ba56988a2629d4db8a2de7 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 21:25:03 +0200 Subject: [PATCH 25/31] fixes --- .../embucket-lambda/extensions}/collector-config/README.md | 4 ++-- .../embucket-lambda/extensions}/collector-config/config.yml | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename {config/layer-root => crates/embucket-lambda/extensions}/collector-config/README.md (84%) rename {config/layer-root => crates/embucket-lambda/extensions}/collector-config/config.yml (100%) diff --git a/config/layer-root/collector-config/README.md b/crates/embucket-lambda/extensions/collector-config/README.md similarity index 84% rename from config/layer-root/collector-config/README.md rename to crates/embucket-lambda/extensions/collector-config/README.md index a0f27f8e..1c235e90 100644 --- a/config/layer-root/collector-config/README.md +++ b/crates/embucket-lambda/extensions/collector-config/README.md @@ -1,7 +1,7 @@ To update the layer config: -1. `cd config/layer-root` +1. `cd crates/embucket-lambda/extensions` 2. `zip -r otel-collector-config-layer.zip collector-config` -3. `cd ../..` +3. `cd ../../..` 4. `aws lambda publish-layer-version --layer-name otel-collector-config --zip-file fileb://config/layer-root/otel-collector-config-layer.zip diff --git a/config/layer-root/collector-config/config.yml b/crates/embucket-lambda/extensions/collector-config/config.yml similarity index 100% rename from config/layer-root/collector-config/config.yml rename to crates/embucket-lambda/extensions/collector-config/config.yml From e5736ca91aa822a1bd422fa8a84b419219d3e85e Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 21:25:13 +0200 Subject: [PATCH 26/31] fixes --- .../embucket-lambda/extensions/collector-config/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/embucket-lambda/extensions/collector-config/README.md b/crates/embucket-lambda/extensions/collector-config/README.md index 1c235e90..372b8400 100644 --- a/crates/embucket-lambda/extensions/collector-config/README.md +++ b/crates/embucket-lambda/extensions/collector-config/README.md @@ -6,8 +6,4 @@ To update the layer config: --layer-name otel-collector-config --zip-file fileb://config/layer-root/otel-collector-config-layer.zip --compatible-runtimes provided.al2 provided.al2023 ---compatible-architectures arm64` - - - - +--compatible-architectures arm64` \ No newline at end of file From d45b05ff077f36d80552d6f3c57286f332fcdee0 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 21:30:22 +0200 Subject: [PATCH 27/31] fixes --- crates/embucket-lambda/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index bc2145f9..ef84baed 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -20,7 +20,7 @@ cd crates/embucket-lambda make deploy # Deploy to a different function -make deploy my-other-function +make deploy FUNCTION_NAME=my-other-function # Deploy without rebuilding make deploy-only From 44138bda33bab607cbabf3dcc8a0fea7ffdcbcda Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 21:54:00 +0200 Subject: [PATCH 28/31] fixes --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 576bb4d8..9a9526cf 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -6,7 +6,7 @@ FUNCTION_NAME ?= embucket-lambda ENV_FILE ?= crates/embucket-lambda/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:21 +OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:22 # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) From 7d11d82dd23069c1bc72823487e9384f44868704 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 22:07:16 +0200 Subject: [PATCH 29/31] fixes --- {crates/embucket-lambda => config}/.env.example | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {crates/embucket-lambda => config}/.env.example (100%) diff --git a/crates/embucket-lambda/.env.example b/config/.env.example similarity index 100% rename from crates/embucket-lambda/.env.example rename to config/.env.example From c1d79ed61be01b15d579005d98458b12c6270673 Mon Sep 17 00:00:00 2001 From: DanCodedThis Date: Tue, 16 Dec 2025 22:07:59 +0200 Subject: [PATCH 30/31] fixes --- crates/embucket-lambda/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 9a9526cf..755f757b 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -3,7 +3,7 @@ # Function name (override with: make deploy FUNCTION_NAME=your-function) FUNCTION_NAME ?= embucket-lambda -ENV_FILE ?= crates/embucket-lambda/.env +ENV_FILE ?= config/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:22 From 7412651646154880a764dbd7a94072d1e6a8b908 Mon Sep 17 00:00:00 2001 From: Yaroslav Litvinov Date: Tue, 16 Dec 2025 23:09:24 +0100 Subject: [PATCH 31/31] add observability --- crates/embucket-lambda/Makefile | 9 +++++++-- crates/embucket-lambda/README.md | 3 +-- crates/embucket-lambda/src/main.rs | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/embucket-lambda/Makefile b/crates/embucket-lambda/Makefile index 755f757b..b4fb3dc6 100644 --- a/crates/embucket-lambda/Makefile +++ b/crates/embucket-lambda/Makefile @@ -6,7 +6,12 @@ FUNCTION_NAME ?= embucket-lambda ENV_FILE ?= config/.env IAM_ROLE ?= arn:aws:iam::767397688925:role/EmbucketLambdaExecRole # --layer-arn required for flexibiity -OTEL_COLLECTOR_LAYER ?= --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1,arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:22 +# otel-collector-config goes second as it overrides original config from opentelemetry-collector +OTEL_COLLECTOR_LAYERS ?= \ + --layer-arn arn:aws:lambda:us-east-2:184161586896:layer:opentelemetry-collector-arm64-0_19_0:1\ + --layer-arn arn:aws:lambda:us-east-2:767397688925:layer:otel-collector-config:22\ + --layer-arn arn:aws:lambda:us-east-2:580247275435:layer:LambdaInsightsExtension-Arm64:33 + # supported features: "streaming" FEATURES_PARAM := $(if $(FEATURES),--features $(FEATURES)) @@ -20,7 +25,7 @@ deploy: build deploy-only # Quick deploy without rebuild deploy-only: @test -n "$(FUNCTION_NAME)" || (echo "ERROR: Missing function name. Use: make deploy " >&2; exit 1) - cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYER) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) + cd ../.. && cargo lambda deploy $(OTEL_COLLECTOR_LAYERS) --iam-role $(IAM_ROLE) --env-file $(ENV_FILE) --binary-name bootstrap $(FUNCTION_NAME) aws logs create-log-group --log-group-name "/aws/lambda/$(FUNCTION_NAME)" >/dev/null 2>&1 || true @$(MAKE) url FUNCTION_NAME=$(FUNCTION_NAME) diff --git a/crates/embucket-lambda/README.md b/crates/embucket-lambda/README.md index ef84baed..c196e54f 100644 --- a/crates/embucket-lambda/README.md +++ b/crates/embucket-lambda/README.md @@ -90,8 +90,7 @@ We send events, spans to stdout log in json format, and in case if AWS X-Ray is #### Exporting telemetry spans to [**honeycomb.io**](https://docs.honeycomb.io/send-data/opentelemetry/collector/) - Required environment variables configuring remote Observability platform: * `HONEYCOMB_API_KEY` - * `HONEYCOMB_DATASET` - * `ENDPOINT_URL` (should become `HONEYCOMB_ENDPOINT_URL`) + * `HONEYCOMB_ENDPOINT_URL` - Optional: * `OTEL_SERVICE_NAME` - `TRACING_LEVEL` - verbosity level, default to "INFO", possible values: "OFF", "ERROR", "WARN", "INFO", "DEBUG", "TRACE". diff --git a/crates/embucket-lambda/src/main.rs b/crates/embucket-lambda/src/main.rs index 8360bcb1..43d06fe8 100644 --- a/crates/embucket-lambda/src/main.rs +++ b/crates/embucket-lambda/src/main.rs @@ -19,8 +19,10 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::BatchSpanProcessor; use opentelemetry_sdk::trace::SdkTracerProvider; +use std::fs; use std::io::IsTerminal; use std::net::{IpAddr, SocketAddr}; +use std::path::Path; use std::sync::Arc; use tower::ServiceExt; use tracing::{error, info}; @@ -39,12 +41,27 @@ type InitResult = Result>; const DISABLED_TARGETS: [&str; 2] = ["h2", "aws_smithy_runtime"]; +fn list_dir(path: &str) -> std::io::Result> { + let mut files = Vec::new(); + for entry in fs::read_dir(Path::new(path))? { + files.push(entry?.path().display().to_string()); + } + Ok(files) +} + #[tokio::main] async fn main() -> Result<(), LambdaError> { let env_config = EnvConfig::from_env(); let tracing_provider = init_tracing_and_logs(&env_config); + // for unknown reason following printed in some strange way + info!( + lambda_configs = ?list_dir("/var/task/config").unwrap_or_default(), + lambda_extensions_configs = ?list_dir("/opt").unwrap_or_default(), + "Lambda files" + ); + info!( data_format = %env_config.data_format, max_concurrency = env_config.max_concurrency_level,