experimental vector build

Author: dervoeti

vector/boil-config.toml

@@ -12,3 +12,18 @@ protoc-version = "31.1"
 # Note: In UBI10, this should change to 4.23.9.0-4.el10
 inotify-tools = "3.22.1.0-1.el9"
 rpm-release = "1"
+
+[versions."d10f3a5b892833232002c97f24bf008dbef1eeec".local-images]
+# We need to point to a special version of stackable-devel which will have the
+# correct toolchain to build vector.
+stackable-devel = "vector-build"
+stackable-base = "1.0.0"
+
+[versions."d10f3a5b892833232002c97f24bf008dbef1eeec".build-arguments]
+# See .scripts/upload_new_protoc_version.sh
+# Unsure which version is used by vector. They seem to install `buf` in workflows.
+protoc-version = "31.1"
+# See here: https://dl.fedoraproject.org/pub/epel/9/Everything/x86_64/Packages/i/
+# Note: In UBI10, this should change to 4.23.9.0-4.el10
+inotify-tools = "3.22.1.0-1.el9"
+rpm-release = "1"

vector/stackable/patches/d10f3a5b892833232002c97f24bf008dbef1eeec/0001-initial-azure_logs_ingestion-implementation.patch

@@ -0,0 +1,60 @@
+From 0a0b5e087e7df29ba1d31094e9cba9bdd6fc3557 Mon Sep 17 00:00:00 2001
+From: Jed Laundry <jlaundry@jlaundry.com>
+Date: Sat, 2 Nov 2024 23:16:15 +0000
+Subject: tidy up error messages
+
+---
+ src/sinks/azure_logs_ingestion/service.rs | 23 ++++++++++++-----------
+ 1 file changed, 12 insertions(+), 11 deletions(-)
+
+diff --git a/src/sinks/azure_logs_ingestion/service.rs b/src/sinks/azure_logs_ingestion/service.rs
+index 9a7d99eca..ad01bbb05 100644
+--- a/src/sinks/azure_logs_ingestion/service.rs
++++ b/src/sinks/azure_logs_ingestion/service.rs
+@@ -137,21 +137,26 @@ impl AzureLogsIngestionService {
+             let res = client.call(request).in_current_span().await?;
+ 
+             if res.status().is_server_error() {
+-                return Err("Server returned a server error".into());
++                return Err("Azure returned a server error".into());
++            }
++
++            if res.status() == StatusCode::UNAUTHORIZED {
++                return Err("Azure returned 401 Unauthorised. Check that the token_scope matches the soverign cloud endpoint.".into());
+             }
+ 
+             if res.status() == StatusCode::FORBIDDEN {
+-                return Err("The service failed to authenticate the request. Verify that the workspace ID and connection key are valid".into());
++                return Err("Azure returned 403 Forbidden. Verify that the credential has the Monitoring Metrics Publisher role on the Data Collection Rule.".into());
+             }
+ 
+             if res.status() == StatusCode::NOT_FOUND {
+-                return Err(
+-                    "Either the URL provided is incorrect, or the request is too large".into(),
+-                );
++                return Err("Azure returned 404 Not Found. Either the URL provided is incorrect, or the request is too large".into());
+             }
+ 
+             if res.status() == StatusCode::BAD_REQUEST {
+-                return Err("The workspace has been closed or the request was invalid".into());
++                let body_bytes: Bytes = hyper::body::to_bytes(res.into_body()).await.unwrap();
++                let body_string: String = String::from_utf8(body_bytes.to_vec()).unwrap();
++                let err_string: String = format!("Azure returned 400 Bad Request: {body_string}");
++                return Err(err_string.into());
+             }
+ 
+             Ok(())
+@@ -176,12 +181,8 @@ impl Service<AzureLogsIngestionRequest> for AzureLogsIngestionService {
+         Box::pin(async move {
+             let http_request = http_request?;
+             let response = client.call(http_request).in_current_span().await?;
+-            let response_status = response.status();
+-            // let body_bytes: Bytes = hyper::body::to_bytes(response.into_body()).await.unwrap();
+-            // let body_string: String = String::from_utf8(body_bytes.to_vec()).unwrap();
+-            // println!("response: {}", body_string);
+             Ok(AzureLogsIngestionResponse {
+-                http_status: response_status,
++                http_status: response.status(),
+                 raw_byte_size: request.metadata.request_encoded_size(),
+                 events_byte_size: request
+                     .metadata

vector/stackable/patches/d10f3a5b892833232002c97f24bf008dbef1eeec/0002-tidy-up-error-messages.patch

@@ -0,0 +1,103 @@
+From 809f9cfe18017b4eafd82d2883a1982b8f80bbf1 Mon Sep 17 00:00:00 2001
+From: Jed Laundry <jlaundry@jlaundry.com>
+Date: Sat, 2 Nov 2024 23:18:16 +0000
+Subject: make token_scope configurable
+
+Signed-off-by: Jed Laundry <jlaundry@jlaundry.com>
+---
+ src/sinks/azure_logs_ingestion/config.rs  | 16 ++++++++++++++--
+ src/sinks/azure_logs_ingestion/service.rs |  5 ++++-
+ 2 files changed, 18 insertions(+), 3 deletions(-)
+
+diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs
+index 5c1249363..5d5a0ef9c 100644
+--- a/src/sinks/azure_logs_ingestion/config.rs
++++ b/src/sinks/azure_logs_ingestion/config.rs
+@@ -22,8 +22,9 @@ use super::{
+ /// Max number of bytes in request body
+ const MAX_BATCH_SIZE: usize = 30 * 1024 * 1024;
+ 
+-// Log Ingestion API version
+-// const API_VERSION: &str = "2023-01-01";
++pub(super) fn default_scope() -> String {
++    "https://monitor.azure.com/.default".into()
++}
+ 
+ /// Configuration for the `azure_logs_ingestion` sink.
+ #[configurable_component(sink(
+@@ -51,6 +52,14 @@ pub struct AzureLogsIngestionConfig {
+     #[configurable(metadata(docs::examples = "Custom-MyTable"))]
+     pub stream_name: String,
+ 
++    /// [Token scope][token_scope] for dedicated Azure regions.
++    ///
++    /// [token_scope]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview
++    #[configurable(metadata(docs::examples = "https://monitor.azure.us/.default"))]
++    #[configurable(metadata(docs::examples = "https://monitor.azure.cn/.default"))]
++    #[serde(default = "default_scope")]
++    pub(super) token_scope: String,
++
+     #[configurable(derived)]
+     #[serde(default, skip_serializing_if = "crate::serde::is_default")]
+     pub encoding: Transformer,
+@@ -81,6 +90,7 @@ impl Default for AzureLogsIngestionConfig {
+             endpoint: Default::default(),
+             dcr: Default::default(),
+             stream_name: Default::default(),
++            token_scope: default_scope(),
+             encoding: Default::default(),
+             batch: Default::default(),
+             request: Default::default(),
+@@ -96,6 +106,7 @@ impl AzureLogsIngestionConfig {
+         &self,
+         cx: SinkContext,
+         endpoint: UriSerde,
++        token_scope: String,
+     ) -> crate::Result<(VectorSink, Healthcheck)> {
+         let endpoint = endpoint.with_default_parts().uri;
+         let protocol = get_http_scheme_from_uri(&endpoint).to_string();
+@@ -117,6 +128,7 @@ impl AzureLogsIngestionConfig {
+             client,
+             endpoint,
+             credential,
++            token_scope,
+         )?;
+         let healthcheck = service.healthcheck();
+ 
+diff --git a/src/sinks/azure_logs_ingestion/service.rs b/src/sinks/azure_logs_ingestion/service.rs
+index ad01bbb05..a4ff46c55 100644
+--- a/src/sinks/azure_logs_ingestion/service.rs
++++ b/src/sinks/azure_logs_ingestion/service.rs
+@@ -75,6 +75,7 @@ pub struct AzureLogsIngestionService {
+     client: HttpClient,
+     endpoint: Uri,
+     credential: Arc<dyn TokenCredential>,
++    token_scope: String,
+     default_headers: HeaderMap,
+ }
+ 
+@@ -84,6 +85,7 @@ impl AzureLogsIngestionService {
+         client: HttpClient,
+         endpoint: Uri,
+         credential: Arc<dyn TokenCredential>,
++        token_scope: String,
+     ) -> crate::Result<Self> {
+         // let mut parts = endpoint.into_parts();
+         // parts.path_and_query = Some(
+@@ -104,6 +106,7 @@ impl AzureLogsIngestionService {
+             client,
+             endpoint,
+             credential,
++            token_scope,
+             default_headers,
+         })
+     }
+@@ -113,7 +116,7 @@ impl AzureLogsIngestionService {
+ 
+         // TODO: make this an option, for soverign clouds
+         let access_token = executor::block_on(self.credential
+-            .get_token(&["https://monitor.azure.com/.default"]))
++            .get_token(&[&self.token_scope]))
+             .expect("failed to get access token from credential");
+         
+         let bearer = format!("Bearer {}", access_token.token.secret());

vector/stackable/patches/d10f3a5b892833232002c97f24bf008dbef1eeec/0003-make-token_scope-configurable.patch

@@ -0,0 +1,120 @@
+From 8acb7b19800288b92380e4a398ca019ed61c93ea Mon Sep 17 00:00:00 2001
+From: Jed Laundry <jlaundry@jlaundry.com>
+Date: Sat, 2 Nov 2024 23:22:14 +0000
+Subject: parameterise the dcr_immutable_id and stream_name
+
+Signed-off-by: Jed Laundry <jlaundry@jlaundry.com>
+---
+ src/sinks/azure_logs_ingestion/config.rs  | 25 +++++++++++++++--------
+ src/sinks/azure_logs_ingestion/service.rs | 22 ++++++++++++--------
+ 2 files changed, 31 insertions(+), 16 deletions(-)
+
+diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs
+index 5d5a0ef9c..711ceae98 100644
+--- a/src/sinks/azure_logs_ingestion/config.rs
++++ b/src/sinks/azure_logs_ingestion/config.rs
+@@ -40,11 +40,11 @@ pub struct AzureLogsIngestionConfig {
+     #[configurable(metadata(docs::examples = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com"))]
+     pub endpoint: String,
+ 
+-    /// The [Data collection rule][dcr] for the Data collection endpoint.
++    /// The [Data collection rule immutable ID][dcr_immutable_id] for the Data collection endpoint.
+     ///
+-    /// [dcr]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview
++    /// [dcr_immutable_id]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/logs-ingestion-api-overview
+     #[configurable(metadata(docs::examples = "dcr-000a00a000a00000a000000aa000a0aa"))]
+-    pub dcr: String,
++    pub dcr_immutable_id: String,
+ 
+     /// The [Stream name][stream_name] for the Data collection rule.
+     ///
+@@ -88,7 +88,7 @@ impl Default for AzureLogsIngestionConfig {
+     fn default() -> Self {
+         Self {
+             endpoint: Default::default(),
+-            dcr: Default::default(),
++            dcr_immutable_id: Default::default(),
+             stream_name: Default::default(),
+             token_scope: default_scope(),
+             encoding: Default::default(),
+@@ -106,6 +106,8 @@ impl AzureLogsIngestionConfig {
+         &self,
+         cx: SinkContext,
+         endpoint: UriSerde,
++        dcr_immutable_id: String,
++        stream_name: String,
+         token_scope: String,
+     ) -> crate::Result<(VectorSink, Healthcheck)> {
+         let endpoint = endpoint.with_default_parts().uri;
+@@ -127,6 +129,8 @@ impl AzureLogsIngestionConfig {
+         let service = AzureLogsIngestionService::new(
+             client,
+             endpoint,
++            dcr_immutable_id,
++            stream_name,
+             credential,
+             token_scope,
+         )?;
+@@ -156,10 +160,15 @@ impl_generate_config_from_default!(AzureLogsIngestionConfig);
+ #[typetag::serde(name = "azure_logs_ingestion")]
+ impl SinkConfig for AzureLogsIngestionConfig {
+     async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> {
+-        // https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com/dataCollectionRules/dcr-000a00a000a00000a000000aa000a0aa/streams/Custom-MyTable?api-version=2023-01-01
+-        // let endpoint = format!("{}/dataCollectionRules/{}/streams/{}", self.endpoint, self.dcr, self.stream_name).parse()?;
+-        let endpoint = self.endpoint.parse()?;
+-        self.build_inner(cx, endpoint).await
++
++        let endpoint: UriSerde = self.endpoint.parse()?;
++        self.build_inner(
++            cx,
++            endpoint,
++            self.dcr_immutable_id.clone(),
++            self.stream_name.clone(),
++            self.token_scope.clone(),
++        ).await
+     }
+ 
+     fn input(&self) -> Input {
+diff --git a/src/sinks/azure_logs_ingestion/service.rs b/src/sinks/azure_logs_ingestion/service.rs
+index a4ff46c55..757ccc5a7 100644
+--- a/src/sinks/azure_logs_ingestion/service.rs
++++ b/src/sinks/azure_logs_ingestion/service.rs
+@@ -15,7 +15,10 @@ use tracing::Instrument;
+ 
+ use crate::{http::HttpClient, sinks::prelude::*};
+ 
+-// JSON content type of logs
++/// Log Ingestion API version
++const API_VERSION: &str = "2023-01-01";
++
++/// JSON content type of logs
+ const CONTENT_TYPE: &str = "application/json";
+ 
+ static CONTENT_TYPE_VALUE: LazyLock<HeaderValue> =
+@@ -84,16 +87,19 @@ impl AzureLogsIngestionService {
+     pub fn new(
+         client: HttpClient,
+         endpoint: Uri,
++        dcr_immutable_id: String,
++        stream_name: String,
+         credential: Arc<dyn TokenCredential>,
+         token_scope: String,
+     ) -> crate::Result<Self> {
+-        // let mut parts = endpoint.into_parts();
+-        // parts.path_and_query = Some(
+-        //     format!("a9ee8e5b-ed0e-4980-9b9c-15e1f939db7f?api-version={API_VERSION}")
+-        //         .parse()
+-        //         .expect("query should never fail to parse"),
+-        // );
+-        // let endpoint = Uri::from_parts(parts)?;
++        let mut parts = endpoint.into_parts();
++        parts.path_and_query = Some(
++            // https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com/dataCollectionRules/dcr-000a00a000a00000a000000aa000a0aa/streams/Custom-MyTable?api-version=2023-01-01
++            format!("/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version={API_VERSION}")
++                .parse()
++                .expect("path and query should never fail to parse"),
++        );
++        let endpoint = Uri::from_parts(parts)?;
+ 
+         let default_headers = {
+             let mut headers = HeaderMap::new();

vector/stackable/patches/d10f3a5b892833232002c97f24bf008dbef1eeec/0004-parameterise-the-dcr_immutable_id-and-stream_name.patch

@@ -0,0 +1,22 @@
+From 1609a631cc1b2e2b3e1abd556455dcb20a4ad28b Mon Sep 17 00:00:00 2001
+From: Jed Laundry <jlaundry@jlaundry.com>
+Date: Sat, 2 Nov 2024 23:30:57 +0000
+Subject: spelling mistake
+
+---
+ src/sinks/azure_logs_ingestion/service.rs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/sinks/azure_logs_ingestion/service.rs b/src/sinks/azure_logs_ingestion/service.rs
+index 757ccc5a7..7c593c456 100644
+--- a/src/sinks/azure_logs_ingestion/service.rs
++++ b/src/sinks/azure_logs_ingestion/service.rs
+@@ -150,7 +150,7 @@ impl AzureLogsIngestionService {
+             }
+ 
+             if res.status() == StatusCode::UNAUTHORIZED {
+-                return Err("Azure returned 401 Unauthorised. Check that the token_scope matches the soverign cloud endpoint.".into());
++                return Err("Azure returned 401 Unauthorised. Check that the token_scope matches the sovereign cloud endpoint.".into());
+             }
+ 
+             if res.status() == StatusCode::FORBIDDEN {

vector/stackable/patches/d10f3a5b892833232002c97f24bf008dbef1eeec/0005-spelling-mistake.patch

@@ -0,0 +1,22 @@
+From e1a61da6161e1bdcf60d12ae052001caffbc6746 Mon Sep 17 00:00:00 2001
+From: Jed Laundry <jlaundry@jlaundry.com>
+Date: Fri, 17 Jan 2025 22:06:03 +1300
+Subject: add reminder from upstream changes
+
+---
+ src/sinks/azure_logs_ingestion/config.rs | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs
+index 711ceae98..0b2a8f3aa 100644
+--- a/src/sinks/azure_logs_ingestion/config.rs
++++ b/src/sinks/azure_logs_ingestion/config.rs
+@@ -123,6 +123,8 @@ impl AzureLogsIngestionConfig {
+         // https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/identity/azure_identity/CHANGELOG.md
+         let credential: Arc<dyn TokenCredential> = azure_identity::create_credential()?;
+ 
++        // TODO this needs to change with toolchain 1.83.0, as per commit ca084cc (#22068)
++        //let tls_settings = TlsSettings::from_options(self.tls.as_ref())?;
+         let tls_settings = TlsSettings::from_options(&self.tls)?;
+         let client = HttpClient::new(Some(tls_settings), &cx.proxy)?;
+