From 3bdd3bf1978472fabf72b204bb612405fc67f5de Mon Sep 17 00:00:00 2001 From: st-omarkhalid Date: Sat, 25 Apr 2026 22:39:58 -0500 Subject: [PATCH] fix(aws_s3 source, aws_sqs source): cap poll_secs at AWS limit of 20 Closes #22934. Both the `aws_s3` source's `sqs.poll_secs` and the `aws_sqs` source's `poll_secs` configuration fields are forwarded to AWS SQS's `ReceiveMessage` API as the `WaitTimeSeconds` parameter, which AWS hard- caps at 20 seconds. Setting a higher value caused AWS to reject the request and surfaced to users as silent ingestion failure with no error message and no hint in the documentation. Add a `validation(range(max = 20))` constraint to both fields so that: - The auto-generated component reference docs surface the 0-20 range in prose (matching the established pattern for other validated numeric fields, e.g. `decrease_ratio` in adaptive_concurrency). - `vector validate` rejects out-of-range configs before startup. Update the field doc comments to state the AWS-imposed limit explicitly, with a link to the AWS API reference. Regenerate the corresponding component CUE docs via `make generate-docs`. --- changelog.d/22934_sqs_poll_secs_range_validation.fix.md | 3 +++ src/sources/aws_s3/sqs.rs | 7 +++++++ src/sources/aws_sqs/config.rs | 7 +++++++ .../cue/reference/components/sources/generated/aws_s3.cue | 6 ++++++ .../cue/reference/components/sources/generated/aws_sqs.cue | 6 ++++++ 5 files changed, 29 insertions(+) create mode 100644 changelog.d/22934_sqs_poll_secs_range_validation.fix.md diff --git a/changelog.d/22934_sqs_poll_secs_range_validation.fix.md b/changelog.d/22934_sqs_poll_secs_range_validation.fix.md new file mode 100644 index 0000000000000..0100d60841267 --- /dev/null +++ b/changelog.d/22934_sqs_poll_secs_range_validation.fix.md @@ -0,0 +1,3 @@ +Constrained the `aws_s3` source's `sqs.poll_secs` and the `aws_sqs` source's `poll_secs` configuration fields to the AWS-imposed maximum of 20 seconds. These fields map to SQS `ReceiveMessage`'s `WaitTimeSeconds` parameter; previously, values above 20 caused AWS to reject the call, manifesting as silent ingestion failure with no error or hint in the documentation. The field documentation now states the limit explicitly, and `vector validate` rejects out-of-range configurations. + +authors: st-omarkhalid diff --git a/src/sources/aws_s3/sqs.rs b/src/sources/aws_s3/sqs.rs index 9ad5b47be61cc..13ac5984b9d64 100644 --- a/src/sources/aws_s3/sqs.rs +++ b/src/sources/aws_s3/sqs.rs @@ -106,11 +106,18 @@ pub(super) struct Config { /// /// Generally, this should not be changed unless instructed to do so, as if messages are available, /// they are always consumed, regardless of the value of `poll_secs`. + /// + /// Maps to the SQS `ReceiveMessage` `WaitTimeSeconds` parameter, which is + /// [capped at 20 seconds by AWS][aws_docs]. Values above 20 are rejected by AWS and result in + /// silent ingestion failure, so this field is constrained to the same range. + /// + /// [aws_docs]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html // NOTE: We restrict this to u32 for safe conversion to i32 later. // NOTE: This value isn't used as a `Duration` downstream, so we don't bother using `serde_with` #[serde(default = "default_poll_secs")] #[derivative(Default(value = "default_poll_secs()"))] #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(validation(range(max = 20)))] pub(super) poll_secs: u32, /// The visibility timeout to use for messages, in seconds. diff --git a/src/sources/aws_sqs/config.rs b/src/sources/aws_sqs/config.rs index c2f630bf28a8d..33600522e6760 100644 --- a/src/sources/aws_sqs/config.rs +++ b/src/sources/aws_sqs/config.rs @@ -41,12 +41,19 @@ pub struct AwsSqsConfig { /// /// Generally, this should not be changed unless instructed to do so, as if messages are available, /// they are always consumed, regardless of the value of `poll_secs`. + /// + /// Maps to the SQS `ReceiveMessage` `WaitTimeSeconds` parameter, which is + /// [capped at 20 seconds by AWS][aws_docs]. Values above 20 are rejected by AWS and result in + /// silent ingestion failure, so this field is constrained to the same range. + /// + /// [aws_docs]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html // NOTE: We restrict this to u32 for safe conversion to i32 later. // NOTE: This value isn't used as a `Duration` downstream, so we don't bother using `serde_with` #[serde(default = "default_poll_secs")] #[derivative(Default(value = "default_poll_secs()"))] #[configurable(metadata(docs::type_unit = "seconds"))] #[configurable(metadata(docs::human_name = "Poll Wait Time"))] + #[configurable(validation(range(max = 20)))] pub poll_secs: u32, /// The visibility timeout to use for messages, in seconds. diff --git a/website/cue/reference/components/sources/generated/aws_s3.cue b/website/cue/reference/components/sources/generated/aws_s3.cue index 95730d6c68ca6..e495f2af7545b 100644 --- a/website/cue/reference/components/sources/generated/aws_s3.cue +++ b/website/cue/reference/components/sources/generated/aws_s3.cue @@ -871,6 +871,12 @@ generated: components: sources: aws_s3: configuration: { Generally, this should not be changed unless instructed to do so, as if messages are available, they are always consumed, regardless of the value of `poll_secs`. + + Maps to the SQS `ReceiveMessage` `WaitTimeSeconds` parameter, which is + [capped at 20 seconds by AWS][aws_docs]. Values above 20 are rejected by AWS and result in + silent ingestion failure, so this field is constrained to the same range. + + [aws_docs]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html """ required: false type: uint: { diff --git a/website/cue/reference/components/sources/generated/aws_sqs.cue b/website/cue/reference/components/sources/generated/aws_sqs.cue index b52cfcea28a31..60b0c5f42c32a 100644 --- a/website/cue/reference/components/sources/generated/aws_sqs.cue +++ b/website/cue/reference/components/sources/generated/aws_sqs.cue @@ -682,6 +682,12 @@ generated: components: sources: aws_sqs: configuration: { Generally, this should not be changed unless instructed to do so, as if messages are available, they are always consumed, regardless of the value of `poll_secs`. + + Maps to the SQS `ReceiveMessage` `WaitTimeSeconds` parameter, which is + [capped at 20 seconds by AWS][aws_docs]. Values above 20 are rejected by AWS and result in + silent ingestion failure, so this field is constrained to the same range. + + [aws_docs]: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html """ required: false type: uint: {