Skip to content

chore: Log and expose aws services error internal details #769

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zeljkoX merged 3 commits into from
May 19, 2026
Merged

chore: Log and expose aws services error internal details #769

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c4286e1
chore: Log and expose aws services error internal details
zeljkoX May 8, 2026
d57fdad
chore: PR improvements
zeljkoX May 18, 2026
5f1db8b
chore: Improvements
tirumerla May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ target/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
.DS_Store
.serena
.codegraph

config.json

Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ panic = "unwind"
aws-config = { version = "1.6.3", features = ["behavior-version-latest"] }
aws-sdk-kms = "1.91.0"
aws-sdk-sqs = "1.56.0"
aws-smithy-types = "1"
aws-smithy-runtime-api = "1"
actix-web = "4"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
Expand Down
24 changes: 20 additions & 4 deletions src/queues/sqs/backend.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use crate::{
},
models::{DefaultAppState, NetworkType},
queues::QueueBackendType,
utils::{aws_error::DisplayErrorContext, classify_sdk_error},
};
use actix_web::web::ThinData;

Expand Down Expand Up @@ -444,8 +445,13 @@ impl SqsBackend {
}

let response = request.send().await.map_err(|e| {
error!(error = %e, queue_url = %queue_url, "Failed to send message to SQS");
QueueBackendError::SqsError(format!("SendMessage failed: {e}"))
error!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
queue_url = %queue_url,
"Failed to send message to SQS"
);
QueueBackendError::SqsError(format!("SendMessage failed: {}", classify_sdk_error(&e)))
})?;

let message_id = response
Expand Down Expand Up @@ -501,7 +507,12 @@ impl SqsBackend {
match sqs_client.get_queue_url().queue_name(dlq_name).send().await {
Ok(output) => output.queue_url().map(str::to_string),
Err(err) => {
warn!(error = %err, dlq_name = %dlq_name, "Failed to resolve DLQ URL at startup");
warn!(
error.kind = classify_sdk_error(&err),
error.detail = %DisplayErrorContext(&err),
dlq_name = %dlq_name,
"Failed to resolve DLQ URL at startup"
);
None
}
}
Expand Down Expand Up @@ -532,7 +543,12 @@ impl SqsBackend {
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(0),
Err(err) => {
warn!(error = %err, dlq_url = %dlq_url, "Failed to fetch DLQ depth");
warn!(
error.kind = classify_sdk_error(&err),
error.detail = %DisplayErrorContext(&err),
dlq_url = %dlq_url,
"Failed to fetch DLQ depth"
);
0
}
}
Expand Down
43 changes: 28 additions & 15 deletions src/queues/sqs/worker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use crate::{
Job, NotificationSend, RelayerHealthCheck, TokenSwapRequest, TransactionRequest,
TransactionSend, TransactionStatusCheck,
},
utils::{aws_error::DisplayErrorContext, classify_sdk_error},
};

use super::{HandlerError, WorkerContext};
Expand Down Expand Up @@ -341,23 +342,17 @@ async fn run_poll_loop(
Err(e) => {
consecutive_poll_errors = consecutive_poll_errors.saturating_add(1);
let backoff_secs = poll_error_backoff_secs(consecutive_poll_errors);
let (error_kind, error_code, error_message) = match &e {
SdkError::ServiceError(ctx) => {
("service", ctx.err().code(), ctx.err().message())
}
SdkError::DispatchFailure(_) => ("dispatch", None, None),
SdkError::ResponseError(_) => ("response", None, None),
SdkError::TimeoutError(_) => ("timeout", None, None),
_ => ("other", None, None),
let (error_code, error_message) = match &e {
SdkError::ServiceError(ctx) => (ctx.err().code(), ctx.err().message()),
_ => (None, None),
};
error!(
queue_type = ?queue_type,
poller_id = poller_id,
error_kind = error_kind,
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
error_code = error_code.unwrap_or("unknown"),
error_message = error_message.unwrap_or("n/a"),
error = %e,
error_debug = ?e,
consecutive_errors = consecutive_poll_errors,
backoff_secs = backoff_secs,
"Failed to receive messages from SQS, backing off"
Expand Down Expand Up @@ -671,7 +666,8 @@ async fn process_message(
{
error!(
queue_type = ?queue_type,
error = %send_err,
error.kind = classify_sdk_error(&send_err),
error.detail = %DisplayErrorContext(&send_err),
"Failed to re-enqueue message; leaving original for visibility timeout retry"
);
// Fall through — original message will retry after visibility timeout
Expand Down Expand Up @@ -771,8 +767,15 @@ async fn defer_message(
.send()
.await
.map_err(|e| {
error!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
queue_url = %queue_url,
"Failed to defer FIFO message via visibility timeout"
);
QueueBackendError::SqsError(format!(
"Failed to defer FIFO message via visibility timeout: {e}"
"Failed to defer FIFO message via visibility timeout: {}",
classify_sdk_error(&e)
))
})?;

Expand Down Expand Up @@ -800,7 +803,16 @@ async fn defer_message(
);

request.send().await.map_err(|e| {
QueueBackendError::SqsError(format!("Failed to defer scheduled message: {e}"))
error!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
queue_url = %queue_url,
"Failed to defer scheduled message"
);
QueueBackendError::SqsError(format!(
"Failed to defer scheduled message: {}",
classify_sdk_error(&e)
))
})?;

Ok(true)
Expand Down Expand Up @@ -965,7 +977,8 @@ async fn flush_delete_batch(
Err(e) => {
error!(
queue_type = ?queue_type,
error = %e,
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
batch_size = chunk.len(),
"Batch delete API call failed (messages will be redelivered)"
);
Expand Down
56 changes: 47 additions & 9 deletions src/services/aws_kms/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ use crate::{
client_cache::AsyncClientCache, signer::evm::utils::recover_evm_signature_from_der,
},
utils::{
self, derive_ethereum_address_from_der, derive_solana_address_from_der,
derive_stellar_address_from_der,
self, aws_error::DisplayErrorContext, classify_sdk_error, derive_ethereum_address_from_der,
derive_solana_address_from_der, derive_stellar_address_from_der,
},
};
use tracing::debug;
use tracing::{debug, warn};

#[cfg(test)]
use mockall::{automock, mock};
Expand All @@ -68,8 +68,6 @@ pub enum AwsKmsError {
GetError(String),
#[error("AWS KMS signing error: {0}")]
SignError(String),
#[error("AWS KMS permissions error: {0}")]
PermissionError(String),
#[error("AWS KMS public key error: {0}")]
RecoveryError(#[from] utils::Secp256k1Error),
#[error("AWS KMS conversion error: {0}")]
Expand Down Expand Up @@ -280,8 +278,16 @@ impl AwsKmsK256 for AwsKmsClient {
.send()
.await
.map_err(|e| {
warn!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
kms_key_id = %key_id,
operation = "get_public_key_secp256k1",
"AWS KMS get_public_key failed"
);
AwsKmsError::GetError(format!(
"Failed to get secp256k1 public key for key '{key_id}': {e:?}"
"Failed to get secp256k1 public key for key '{key_id}': {}",
classify_sdk_error(&e)
))
Comment thread
zeljkoX marked this conversation as resolved.
})?;

Expand Down Expand Up @@ -316,7 +322,19 @@ impl AwsKmsK256 for AwsKmsClient {

// Process the result, extract DER signature
let der_signature = sign_result
.map_err(|e| AwsKmsError::PermissionError(e.to_string()))?
.map_err(|e| {
warn!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
kms_key_id = %key_id,
operation = "sign_digest_secp256k1",
"AWS KMS sign failed"
);
AwsKmsError::SignError(format!(
"Failed to sign secp256k1 digest for key '{key_id}': {}",
classify_sdk_error(&e)
))
})?
Comment thread
zeljkoX marked this conversation as resolved.
.signature
.ok_or(AwsKmsError::SignError(
"Signature not found in response".to_string(),
Expand Down Expand Up @@ -347,8 +365,16 @@ impl AwsKmsEd25519 for AwsKmsClient {
.send()
.await
.map_err(|e| {
warn!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
kms_key_id = %key_id,
operation = "get_public_key_ed25519",
"AWS KMS get_public_key failed"
);
AwsKmsError::GetError(format!(
"Failed to get Ed25519 public key for key '{key_id}': {e:?}"
"Failed to get Ed25519 public key for key '{key_id}': {}",
classify_sdk_error(&e)
))
})?;

Expand Down Expand Up @@ -386,7 +412,19 @@ impl AwsKmsEd25519 for AwsKmsClient {

// Process the result, extract signature
let signature = sign_result
.map_err(|e| AwsKmsError::SignError(e.to_string()))?
.map_err(|e| {
warn!(
error.kind = classify_sdk_error(&e),
error.detail = %DisplayErrorContext(&e),
kms_key_id = %key_id,
operation = "sign_ed25519",
"AWS KMS sign failed"
);
AwsKmsError::SignError(format!(
"Failed to sign Ed25519 message for key '{key_id}': {}",
classify_sdk_error(&e)
))
})?
.signature
.ok_or(AwsKmsError::SignError(
"Signature not found in response".to_string(),
Expand Down
Loading
Loading