Skip to content

Commit 30d9a58

Browse files
jlaundrythomasqueirozbpront
authored
feat(azure_blob sink): Expand support for Azure authentication types (#24729)
* start by moving auth config to azure_common * add auth to azure_blob Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * initial auth switch Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * integration test WIP Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * doc updates Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add feature changelog Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * clippy changes Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add account_name and blob_endpoint options Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * remove block_in_place * add ClientCertificateCredential Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * remake certificate with required attributes * refactor integration tests to support both connection string and oauth Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * update doc examples Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * make clippy happy Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * fix tests Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * make "client_secret_credential" explicitly configured Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add azure_logs_ingestion to Azure integration tests Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * Err out the other TlsConfig options Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * Add warning message to connection_string Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * Simplify TlsConfig Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add workload identity config options * Move warning into docs::warnings * Properly regenerate cue file * Fix spaces * tidy options creation Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add UAMI type option Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add note about breaking change * propagate credential errors instead of panicking * automatic Cargo lock change Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * clarify PEM cert file Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * action test fixes Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * allow sinks-azure_logs_ingestion to build without sinks-azure_blob Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * require auth when using account_name or blob_endpoint Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * add dep:base64 Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> * check-fmt changes * space diff fix --------- Signed-off-by: Jed Laundry <jlaundry@jlaundry.com> Co-authored-by: Thomas <thomas.schneider@datadoghq.com> Co-authored-by: Pavlos Rontidis <pavlos.rontidis@gmail.com>
1 parent dcf98a7 commit 30d9a58

File tree

28 files changed

+1542
-342
lines changed

28 files changed

+1542
-342
lines changed

‎.github/actions/spelling/allow.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ greptimedb
210210
GSM
211211
gvisor
212212
gws
213+
gzip'd
213214
hadoop
214215
Haier
215216
Haipad
@@ -460,6 +461,7 @@ rumqttc
460461
Salesforce
461462
Samsung
462463
SBT
464+
schema'd
463465
scriptblock
464466
Sega
465467
Segoe

‎.github/actions/spelling/expect.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,7 @@ mycie
379379
mycorp
380380
mydatabase
381381
mylabel
382+
mylogstorage
382383
mypod
383384
myvalue
384385
Namazu

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ aws-smithy-types = { version = "1.2.11", default-features = false, features = ["
302302

303303
# Azure
304304
azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true }
305-
azure_identity = { version = "0.30", optional = true }
305+
azure_identity = { version = "0.30", features = ["client_certificate"], optional = true }
306306

307307
# Azure Storage
308308
azure_storage_blob = { version = "0.7", optional = true }
@@ -875,8 +875,8 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"]
875875
sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"]
876876
sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"]
877877
sinks-axiom = ["sinks-http"]
878-
sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"]
879-
sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity"]
878+
sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"]
879+
sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"]
880880
sinks-azure_monitor_logs = []
881881
sinks-blackhole = []
882882
sinks-chronicle = []
@@ -987,7 +987,8 @@ aws-integration-tests = [
987987
]
988988

989989
azure-integration-tests = [
990-
"azure-blob-integration-tests"
990+
"azure-blob-integration-tests",
991+
"azure-logs-ingestion-integration-tests",
991992
]
992993

993994
aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"]
@@ -1001,6 +1002,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"]
10011002
aws-sns-integration-tests = ["sinks-aws_sns"]
10021003
axiom-integration-tests = ["sinks-axiom"]
10031004
azure-blob-integration-tests = ["sinks-azure_blob"]
1005+
azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"]
10041006
chronicle-integration-tests = ["sinks-gcp"]
10051007
clickhouse-integration-tests = ["sinks-clickhouse"]
10061008
databend-integration-tests = ["sinks-databend"]

‎changelog.d/24729_azure_blob_authentication.feature.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types.
2+
3+
authors: jlaundry

‎changelog.d/24729_azure_logs_ingestion.breaking.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` to your sink config (this was previously the default, and now must be explicitly configured).
2+
3+
authors: jlaundry

‎src/sinks/azure_blob/config.rs‎

Lines changed: 83 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ use crate::{
1616
sinks::{
1717
Healthcheck, VectorSink,
1818
azure_common::{
19-
self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink,
19+
self, config::AzureAuthentication, config::AzureBlobRetryLogic,
20+
config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink,
2021
},
2122
util::{
2223
BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt,
@@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults {
4142
#[derive(Clone, Debug)]
4243
#[serde(deny_unknown_fields)]
4344
pub struct AzureBlobSinkConfig {
45+
#[configurable(derived)]
46+
#[serde(default)]
47+
pub auth: Option<AzureAuthentication>,
48+
4449
/// The Azure Blob Storage Account connection string.
4550
///
4651
/// Authentication with an access key or shared access signature (SAS)
@@ -55,13 +60,33 @@ pub struct AzureBlobSinkConfig {
5560
/// | Allowed services | Blob |
5661
/// | Allowed resource types | Container & Object |
5762
/// | Allowed permissions | Read & Create |
63+
#[configurable(metadata(
64+
docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \
65+
resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \
66+
connection strings secure and not expose them in logs, error messages, or version control systems."
67+
))]
5868
#[configurable(metadata(
5969
docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net"
6070
))]
6171
#[configurable(metadata(
6272
docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken"
6373
))]
64-
pub connection_string: SensitiveString,
74+
#[configurable(metadata(docs::examples = "AccountName=mylogstorage"))]
75+
pub connection_string: Option<SensitiveString>,
76+
77+
/// The Azure Blob Storage Account name.
78+
///
79+
/// If provided, this will be used instead of the `connection_string`.
80+
/// This is useful for authenticating with an Azure credential.
81+
#[configurable(metadata(docs::examples = "mylogstorage"))]
82+
pub(super) account_name: Option<String>,
83+
84+
/// The Azure Blob Storage endpoint.
85+
///
86+
/// If provided, this will be used instead of the `connection_string`.
87+
/// This is useful for authenticating with an Azure credential.
88+
#[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))]
89+
pub(super) blob_endpoint: Option<String>,
6590

6691
/// The Azure Blob Storage Account container name.
6792
#[configurable(metadata(docs::examples = "my-logs"))]
@@ -138,6 +163,10 @@ pub struct AzureBlobSinkConfig {
138163
skip_serializing_if = "crate::serde::is_default"
139164
)]
140165
pub(super) acknowledgements: AcknowledgementsConfig,
166+
167+
#[configurable(derived)]
168+
#[serde(default)]
169+
pub tls: Option<AzureBlobTlsConfig>,
141170
}
142171

143172
pub fn default_blob_prefix() -> Template {
@@ -147,7 +176,10 @@ pub fn default_blob_prefix() -> Template {
147176
impl GenerateConfig for AzureBlobSinkConfig {
148177
fn generate_config() -> toml::Value {
149178
toml::Value::try_from(Self {
150-
connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(),
179+
auth: None,
180+
connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()),
181+
account_name: None,
182+
blob_endpoint: None,
151183
container_name: String::from("logs"),
152184
blob_prefix: default_blob_prefix(),
153185
blob_time_format: Some(String::from("%s")),
@@ -157,6 +189,7 @@ impl GenerateConfig for AzureBlobSinkConfig {
157189
batch: BatchConfig::default(),
158190
request: TowerRequestConfig::default(),
159191
acknowledgements: Default::default(),
192+
tls: None,
160193
})
161194
.unwrap()
162195
}
@@ -166,11 +199,56 @@ impl GenerateConfig for AzureBlobSinkConfig {
166199
#[typetag::serde(name = "azure_blob")]
167200
impl SinkConfig for AzureBlobSinkConfig {
168201
async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> {
202+
let connection_string: String = match (
203+
&self.connection_string,
204+
&self.account_name,
205+
&self.blob_endpoint,
206+
) {
207+
(Some(connstr), None, None) => connstr.inner().into(),
208+
(None, Some(account_name), None) => {
209+
if self.auth.is_none() {
210+
return Err(
211+
"`auth` configuration must be provided when using `account_name`".into(),
212+
);
213+
}
214+
format!("AccountName={}", account_name)
215+
}
216+
(None, None, Some(blob_endpoint)) => {
217+
if self.auth.is_none() {
218+
return Err(
219+
"`auth` configuration must be provided when using `blob_endpoint`".into(),
220+
);
221+
}
222+
// BlobEndpoint must always end in a trailing slash
223+
let blob_endpoint = if blob_endpoint.ends_with('/') {
224+
blob_endpoint.clone()
225+
} else {
226+
format!("{}/", blob_endpoint)
227+
};
228+
format!("BlobEndpoint={}", blob_endpoint)
229+
}
230+
(None, None, None) => {
231+
return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into());
232+
}
233+
(Some(_), Some(_), _) => {
234+
return Err("Cannot provide both `connection_string` and `account_name`".into());
235+
}
236+
(Some(_), _, Some(_)) => {
237+
return Err("Cannot provide both `connection_string` and `blob_endpoint`".into());
238+
}
239+
(_, Some(_), Some(_)) => {
240+
return Err("Cannot provide both `account_name` and `blob_endpoint`".into());
241+
}
242+
};
243+
169244
let client = azure_common::config::build_client(
170-
self.connection_string.clone().into(),
245+
self.auth.clone(),
246+
connection_string.clone(),
171247
self.container_name.clone(),
172248
cx.proxy(),
173-
)?;
249+
self.tls.clone(),
250+
)
251+
.await?;
174252

175253
let healthcheck = azure_common::config::build_healthcheck(
176254
self.container_name.clone(),

0 commit comments

Comments
 (0)