diff --git a/Cargo.lock b/Cargo.lock index 462d4872dc..1e39db3652 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,12 @@ dependencies = [ "regex", ] +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.2" @@ -258,7 +264,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", - "event-listener", + "event-listener 2.5.3", "futures-core", ] @@ -275,6 +281,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-compression" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93c1f86859c1af3d514fa19e8323147ff10ea98684e6c7b307912509f50e67b2" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -350,7 +380,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "hex", "http 0.2.12", "ring 0.17.14", @@ -389,7 +419,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "http-body 0.4.6", "percent-encoding", @@ -416,7 +446,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand", + "fastrand 2.3.0", "flate2", "http 0.2.12", "http-body 0.4.6", @@ -444,7 +474,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "hex", "hmac", "http 0.2.12", @@ -761,7 +791,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", @@ -941,6 +971,165 @@ dependencies = [ "tracing", ] +[[package]] +name = "azservicebus" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee87ee5702a4a33f760859859b15a80dbdd666871e6f6209b945910bb81bb8db" +dependencies = [ + "azure_core 0.25.0", + "base64 0.22.1", + "const_format", + "digest", + "fe2o3-amqp", + "fe2o3-amqp-cbs", + "fe2o3-amqp-management", + "fe2o3-amqp-types", + "fe2o3-amqp-ws", + "fluvio-wasm-timer", + "futures-util", + "getrandom 0.2.15", + "hmac", + "indexmap 2.7.0", + "js-sys", + "log", + "rand 0.8.5", + "serde", + "serde_amqp", + "sha2", + "thiserror 1.0.69", + "time", + "timer-kit", + "tokio", + "tokio-util", + "url", + "urlencoding", + "uuid", +] + +[[package]] +name = "azure_core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.15", + "hmac", + "http-types", + "once_cell", + "paste", + "pin-project", + "quick-xml", + "rand 0.8.5", + "reqwest 0.12.23", + "rustc_version", + "serde", + "serde_json", + "sha2", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_core" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c33c072c9d87777262f35abfe2a64b609437076551d4dac8373e60f0e3fde9" +dependencies = [ + "async-lock", + "async-trait", + "bytes", + "futures", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_identity" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb64e97087965481c94f1703c57e678df09df73e2cdaee8952558f9c6c7d100" +dependencies = [ + "async-lock", + "async-trait", + "azure_core 0.25.0", + "futures", + "pin-project", + "serde", + "time", + "tracing", + "typespec_client_core", + "url", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +dependencies = [ + "RustyXML", + "async-lock", + "async-trait", + "azure_core 0.21.0", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core 0.21.0", + "azure_storage", + "azure_svc_blobstorage", + "bytes", + "futures", + "serde", + "serde_derive", + "serde_json", + "time", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +dependencies = [ + "azure_core 0.21.0", + "bytes", + "futures", + "log", + "once_cell", + "serde", + "serde_json", + "time", +] + [[package]] name = "backoff" version = "0.2.1" @@ -957,7 +1146,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand", + "fastrand 2.3.0", ] [[package]] @@ -1423,6 +1612,23 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "compression-codecs" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680dc087785c5230f8e8843e2e57ac7c1c90488b6a91b88caa265410568f441b" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1460,6 +1666,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const_format" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1909,6 +2135,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecdsa" version = "0.14.8" @@ -2013,6 +2245,11 @@ dependencies = [ "aws-sdk-sqs", "aws-smithy-types", "axum 0.7.9", + "azservicebus", + "azure_core 0.25.0", + "azure_identity", + "azure_storage", + "azure_storage_blobs", "backtrace", "base32", "base64 0.21.7", @@ -2030,6 +2267,7 @@ dependencies = [ "duct", "email_address", "env_logger 0.10.2", + "fe2o3-amqp-types", "flate2", "form_urlencoded", "futures", @@ -2090,6 +2328,7 @@ dependencies = [ "subtle", "sysinfo", "thiserror 1.0.69", + "time", "tokio", "tokio-nsq", "tokio-postgres", @@ -2267,18 +2506,130 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.1", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fe2o3-amqp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a579ef4f1fb186f04bcdc9caf0c335adedebe879227c96d56876d473aa3d20a" +dependencies = [ + "bytes", + "fe2o3-amqp-types", + "futures-util", + "getrandom 0.3.3", + "native-tls", + "parking_lot 0.12.3", + "pin-project-lite", + "serde", + "serde_amqp", + "serde_bytes", + "slab", + "thiserror 2.0.10", + "tokio", + "tokio-native-tls", + "tokio-stream", + "tokio-util", + "url", + "wasmtimer", +] + +[[package]] +name = "fe2o3-amqp-cbs" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cae904b214ffa3c9bae26e4129d300fe79189d2ef70503071fb25ff9127531e" +dependencies = [ + "fe2o3-amqp", + "fe2o3-amqp-management", + "trait-variant", +] + +[[package]] +name = "fe2o3-amqp-management" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0582084762bdf022540c37868a0808e9f54dbcc51fe56f6212da59c167569cda" +dependencies = [ + "fe2o3-amqp", + "fe2o3-amqp-types", + "serde", + "thiserror 2.0.10", +] + +[[package]] +name = "fe2o3-amqp-types" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bcc8d13ed13fbb2fb664a6df114bcc32f8ca85c9cb6b89d4e7576c47f583706" +dependencies = [ + "ordered-float", + "serde", + "serde_amqp", + "serde_bytes", + "serde_repr", +] + +[[package]] +name = "fe2o3-amqp-ws" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117053be08403ac3b36538bf5a4cf42d328cdcf09950142524dd9ca4b25121a" +dependencies = [ + "bytes", + "futures-util", + "getrandom 0.3.3", + "http 1.2.0", + "js-sys", + "pin-project-lite", + "thiserror 2.0.10", + "tokio", + "tokio-tungstenite 0.26.2", + "tungstenite 0.26.2", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "ff" version = "0.12.1" @@ -2327,6 +2678,21 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluvio-wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b768c170dc045fa587a8f948c91f9bcfb87f774930477c6215addf54317f137f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2450,13 +2816,28 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-lite" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -3167,6 +3548,26 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite 1.13.0", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.9.5" @@ -3539,6 +3940,12 @@ dependencies = [ "serde", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "insta" version = "1.42.0" @@ -4027,7 +4434,7 @@ name = "miniredis-rs" version = "0.1.0" dependencies = [ "bytes", - "futures-lite", + "futures-lite 2.6.1", "miniredis-rs", "mlua", "ordered-float", @@ -4451,6 +4858,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" dependencies = [ "num-traits", + "rand 0.8.5", + "serde", ] [[package]] @@ -5419,6 +5828,16 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quickcheck" version = "1.0.3" @@ -5541,6 +5960,7 @@ dependencies = [ "libc", "rand_chacha 0.3.1", "rand_core 0.6.4", + "serde", ] [[package]] @@ -5614,6 +6034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom 0.2.15", + "serde", ] [[package]] @@ -5820,6 +6241,7 @@ version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ + "async-compression 0.4.33", "base64 0.22.1", "bytes", "encoding_rs", @@ -6384,6 +6806,45 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_amqp" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e76738e7a058df01b5b33194359930a1aa5bc233dc07e510f458aca47189aea2" +dependencies = [ + "bytes", + "indexmap 2.7.0", + "ordered-float", + "serde", + "serde_amqp_derive", + "serde_bytes", + "thiserror 2.0.10", + "time", +] + +[[package]] +name = "serde_amqp_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22da57ecf44834259b4416250608e11da620750be91305bf6ae5d398954ddc6d" +dependencies = [ + "convert_case", + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.95", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -6436,6 +6897,28 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -7243,7 +7726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.3.0", "getrandom 0.2.15", "once_cell", "rustix 0.38.43", @@ -7333,6 +7816,7 @@ checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "itoa", + "js-sys", "num-conv", "powerfmt", "serde", @@ -7356,6 +7840,20 @@ dependencies = [ "time-core", ] +[[package]] +name = "timer-kit" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1323065b94fee01a4049c46c671f87d4aef531d3d08f7ebe427ad07bb19a5b" +dependencies = [ + "fluvio-wasm-timer", + "futures-util", + "pin-project-lite", + "slab", + "thiserror 1.0.69", + "tokio", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -7437,7 +7935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "289e54c5548b30d6fd1edb525812fa26c745ba0dccdf5fc552ffe7f8b0f7991e" dependencies = [ "anyhow", - "async-compression", + "async-compression 0.3.15", "backoff", "built", "byteorder", @@ -7601,6 +8099,20 @@ dependencies = [ "tungstenite 0.24.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite 0.26.2", +] + [[package]] name = "tokio-util" version = "0.7.13" @@ -7613,6 +8125,7 @@ dependencies = [ "futures-util", "hashbrown 0.14.5", "pin-project-lite", + "slab", "tokio", ] @@ -7854,6 +8367,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "triomphe" version = "0.1.14" @@ -7921,6 +8445,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http 1.2.0", + "httparse", + "log", + "native-tls", + "rand 0.9.2", + "sha1", + "thiserror 2.0.10", + "utf-8", +] + [[package]] name = "txtar" version = "1.0.0" @@ -7950,6 +8492,56 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "typespec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c7a952f1f34257f945fc727b20defe7a3c01c05ddd42925977626cfa6e62ab" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5879ce67ba9e525fe088c882ede1337c32c3f80e83e72d9fd3cc6c8e05bcb3d7" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.15", + "pin-project", + "rand 0.8.5", + "reqwest 0.12.23", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbccdbe531c8d553812a609bdb70c0d1002ad91333498e18df42c98744b15cc" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.95", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -8013,6 +8605,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -8040,6 +8638,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -8164,6 +8763,12 @@ dependencies = [ "libc", ] +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "walkdir" version = "2.5.0" @@ -8310,6 +8915,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmtimer" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.12.3", + "pin-utils", + "slab", + "wasm-bindgen", +] + [[package]] name = "web-sys" version = "0.3.81" diff --git a/docs/platform/infrastructure/azure-config-reference.md b/docs/platform/infrastructure/azure-config-reference.md new file mode 100644 index 0000000000..c59568164a --- /dev/null +++ b/docs/platform/infrastructure/azure-config-reference.md @@ -0,0 +1,345 @@ +--- +seotitle: Azure Infrastructure Config Reference — Encore Self-Hosting +seodesc: Reference documentation for Azure-specific infra config JSON fields used when self-hosting Encore on Azure +title: Azure Config Reference +subtitle: Runtime configuration fields for self-hosting Encore on Azure +lang: platform +--- + +This page is a reference for the Azure-specific fields in Encore's runtime infrastructure configuration JSON. These fields are used when **self-hosting** Encore on Azure — for example after running `encore eject` — and are not required when using Encore Cloud managed deployments. + +For the overall structure of the infrastructure config see the [infrastructure configuration guide][infra-config]. + +--- + +## `AzureServiceBusProvider` — Pub/Sub + +Configures [Azure Service Bus][az-servicebus] as the pub/sub backend. Set this inside a `PubsubProvider` entry in the runtime `pubsub_providers` array. + +**Source:** `runtimes/go/appruntime/exported/config/config.go` → `AzureServiceBusProvider` +**Proto:** `proto/encore/runtime/v1/infra.proto` → `PubSubProvider.AzureServiceBus` ✅ (implemented) + +### Fields + +| Field | Type | Required | Description | +|---|---|---|---| +| `namespace` | `string` | ✅ | The fully-qualified Azure Service Bus namespace hostname, e.g. `my-namespace.servicebus.windows.net`. | + +Authentication uses **DefaultAzureCredential** — managed identity in production, Azure CLI or environment credentials locally. + +### Example + +```json +{ + "pubsub_providers": [ + { + "azure": { + "namespace": "my-namespace.servicebus.windows.net" + } + } + ], + "pubsub_topics": { + "user-events": { + "encore_name": "user-events", + "provider_id": 0, + "provider_name": "user-events", + "subscriptions": { + "email-service": { + "id": "email-service", + "encore_name": "email-service", + "provider_name": "user-events~email-service" + } + } + } + } +} +``` + +> **Tip:** The `provider_name` for a topic maps to the Azure Service Bus **topic** name, and the subscription's `provider_name` maps to the **subscription** name within that topic (Azure Service Bus subscription names conventionally use the `~` pattern, but the exact names are whatever you provision in Azure). + +--- + +## `AzureBlobBucketProvider` — Object Storage + +Configures [Azure Blob Storage][az-blob] as the object storage backend. Set this inside a `BucketProvider` entry in the runtime `bucket_providers` array. + +**Source:** `runtimes/go/appruntime/exported/config/config.go` → `AzureBlobBucketProvider` +**Proto:** `proto/encore/runtime/v1/infra.proto` → `BucketProvider.AzBlob` ✅ (implemented) + +> ✅ **Proto gap resolved:** The `AzBlob` message exists in `infra.proto` and the Go parsing layer that maps `BucketCluster.az_blob` → `config.Runtime.BucketProviders[].AzureBlob` has now been implemented (fixed this sprint by Neo). Self-hosted deployments using `infra.proto` can now activate Azure Blob Storage. + +### Fields + +| Field | Type | Required | Description | +|---|---|---|---| +| `storage_account` | `string` | ✅ | The name of the Azure storage account (e.g. `myappstgprod`). | +| `connection_string` | `string \| null` | ☐ | Full Azure Blob Storage connection string. When set it takes precedence over `storage_account` + `storage_key`. The account name and key embedded in the string are also used for SAS URL generation. | +| `storage_key` | `string \| null` | ☐ | Azure storage account key for SharedKey authentication. Required if you need to generate signed (SAS) URLs. If both `connection_string` and `storage_key` are omitted, **DefaultAzureCredential** (managed identity) is used for authentication. | + +> **Note:** In production on AKS or Container Apps with managed identity, omit both `connection_string` and `storage_key`. The runtime will authenticate using the pod/container's managed identity, which should be granted `Storage Blob Data Contributor` (or `Reader`) on the relevant containers. + +### Example — Managed Identity (recommended for production) + +```json +{ + "bucket_providers": [ + { + "azure_blob": { + "storage_account": "myappstgprod" + } + } + ], + "buckets": { + "profile-images": { + "cluster_id": 0, + "encore_name": "profile-images", + "cloud_name": "profile-images-a1b2c3", + "key_prefix": "", + "public_base_url": "https://myappstgprod.blob.core.windows.net/profile-images-a1b2c3" + } + } +} +``` + +### Example — Explicit Storage Key + +```json +{ + "bucket_providers": [ + { + "azure_blob": { + "storage_account": "myappstgprod", + "storage_key": "base64encodedkey==" + } + } + ], + "buckets": { + "uploads": { + "cluster_id": 0, + "encore_name": "uploads", + "cloud_name": "uploads-d4e5f6", + "key_prefix": "" + } + } +} +``` + +--- + +## `AzureMonitorMetricsProvider` — Metrics + +Configures [Azure Monitor custom metrics][az-monitor-custom] as the metrics export backend. Set this as the `azure_monitor` field on the `Metrics` object in the runtime config. + +**Source:** `runtimes/go/appruntime/exported/config/config.go` → `AzureMonitorMetricsProvider` +**Proto:** `proto/encore/runtime/v1/runtime.proto` → `MetricsProvider.AzureMonitor` ✅ (implemented) + +> ⚠️ **Proto gap:** `MetricsProvider.AzureMonitor` is not yet defined in `proto/encore/runtime/v1/infra.proto` — it is being added this sprint by The Keymaker. Until that change ships, Azure Monitor metrics config is only available via the `runtime.proto` path (Encore Cloud hosted deployments). For self-hosted deployments, configure metrics via the `runtime.proto` `MetricsProvider.AzureMonitor` message directly rather than through `infra.proto`. + +### Fields + +| Field | Type | Required | Description | +|---|---|---|---| +| `location` | `string` | ✅ | Azure region of the target resource (e.g. `eastus`, `westeurope`). | +| `subscription_id` | `string` | ✅ | Azure subscription ID that owns the resource. | +| `resource_group` | `string` | ✅ | Resource group that contains the target resource. | +| `resource_namespace` | `string` | ✅ | Resource provider namespace and type, e.g. `Microsoft.ContainerService/managedClusters` or `Microsoft.App/containerApps`. | +| `resource_name` | `string` | ✅ | Name of the target Azure resource (the AKS cluster name, Container App name, etc.). | +| `namespace` | `string` | ✅ | Custom metrics namespace that Encore will write to in Azure Monitor (e.g. `Encore/App`). | + +Authentication uses **DefaultAzureCredential**. In production the managed identity must be granted the `Monitoring Metrics Publisher` role on the target resource. + +### Example + +```json +{ + "metrics": { + "collection_interval": 15000000000, + "azure_monitor": { + "location": "eastus", + "subscription_id": "00000000-0000-0000-0000-000000000000", + "resource_group": "my-app-prod-rg", + "resource_namespace": "Microsoft.ContainerService/managedClusters", + "resource_name": "my-app-prod-aks", + "namespace": "Encore/App" + } + } +} +``` + +> **Note:** `collection_interval` is expressed in nanoseconds. `15000000000` = 15 seconds. + +--- + +## `AzureKeyVaultSecretsProvider` — Secrets + +Configures [Azure Key Vault][az-keyvault] as the remote secrets backend. Set this as the `secrets_provider.azure_key_vault` field in the **InfraConfig** (the self-hosting configuration JSON, distinct from the runtime config). + +**Source:** `runtimes/go/appruntime/exported/config/infra/config.go` → `AzureKeyVaultSecretsProvider` +**Proto:** Not yet present in `infra.proto` — configured exclusively via the JSON InfraConfig. A proto definition will be added in a future release. + +> ⚠️ **Proto gap:** `SecretsProvider` (and `AzureKeyVaultSecretsProvider`) are not yet defined in `proto/encore/runtime/v1/infra.proto`. Until that is resolved, the Key Vault secrets provider is only available through the JSON-based `InfraConfig` used in the self-hosting / eject flow. Encore Cloud managed deployments configure secrets automatically. + +### Fields + +| Field | Type | Required | Description | +|---|---|---|---| +| `vault_url` | `string` | ✅ | Base URL of the Azure Key Vault, e.g. `https://my-vault.vault.azure.net/`. | + +Secret names in the Encore application map **directly** to secret names in the Key Vault. Authentication uses **DefaultAzureCredential** — managed identity in production (the identity must be granted `Key Vault Secrets User` on the vault), Azure CLI credentials locally. + +### Example (InfraConfig) + +```json +{ + "metadata": { + "app_id": "my-app", + "env_name": "production", + "env_type": "production", + "cloud": "azure", + "base_url": "https://api.my-app.example.com" + }, + "secrets_provider": { + "azure_key_vault": { + "vault_url": "https://my-app-prod-kv.vault.azure.net/" + } + } +} +``` + +--- + +## `AzureMetadata` — IMDS Collector + +When an Encore application starts on Azure, the runtime automatically queries the [Azure Instance Metadata Service (IMDS)][az-imds] at `http://169.254.169.254/metadata/instance?api-version=2021-02-01` to enrich traces and logs with cloud context. + +**Source:** `runtimes/go/appruntime/infrasdk/metadata/azure_collector.go` +**Proto:** Not a configurable field — the collector is registered automatically when `env_cloud` is `"azure"`. + +### Fields collected from IMDS + +| IMDS field | Mapped to | Notes | +|---|---|---| +| `compute.location` | Azure region (e.g. `eastus`) | Used for metrics and tracing context | +| `compute.resourceGroupName` | `ServiceID` in container metadata | Closest equivalent to an ECS service boundary | +| `compute.vmId` | `InstanceID` (last 8 chars) | Unique instance identifier for tracing | +| `compute.name` | VM / node name | Available but not currently surfaced in traces | +| `compute.subscriptionId` | Subscription context | Available but not currently surfaced in traces | + +> **Note:** The IMDS endpoint is only reachable from within an Azure VM or container. Outside of Azure the collector returns empty metadata gracefully — it does not fail startup. + +### Enabling the IMDS collector + +No configuration is required. Set `cloud` to `"azure"` in the `metadata` block of your `InfraConfig` and the collector activates automatically: + +```json +{ + "metadata": { + "app_id": "my-app", + "env_name": "production", + "env_type": "production", + "cloud": "azure" + } +} +``` + +To **disable** the Azure IMDS collector at compile time (e.g. to reduce binary size in a non-Azure deployment), build your application with the `encore_no_azure` build tag: + +```bash +go build -tags encore_no_azure ./... +``` + +--- + +## Full Self-Hosting Example + +The following shows a complete `InfraConfig` JSON for a self-hosted Encore app on Azure using all four Azure providers: + +```json +{ + "metadata": { + "app_id": "my-app", + "env_name": "production", + "env_type": "production", + "cloud": "azure", + "base_url": "https://api.my-app.example.com" + }, + "secrets_provider": { + "azure_key_vault": { + "vault_url": "https://my-app-prod-kv.vault.azure.net/" + } + }, + "metrics": { + "collection_interval": 15000000000, + "azure_monitor": { + "location": "eastus", + "subscription_id": "00000000-0000-0000-0000-000000000000", + "resource_group": "my-app-prod-rg", + "resource_namespace": "Microsoft.ContainerService/managedClusters", + "resource_name": "my-app-prod-aks", + "namespace": "Encore/App" + } + }, + "sql_servers": [ + { + "host": "my-app-prod-pg.postgres.database.azure.com:5432", + "tls_config": { + "disable_tls_hostname_verification": false + }, + "databases": { + "users": { + "name": "users", + "username": { "value": "encore_users" }, + "password": { "$env": "DB_USERS_PASSWORD" } + } + } + } + ], + "redis": { + "sessions": { + "host": "my-app-prod-redis.redis.cache.windows.net:6380", + "database_index": 0, + "auth": { + "type": "auth_string", + "auth_string": { "$env": "REDIS_AUTH_STRING" } + }, + "tls_config": {} + } + }, + "pubsub": [ + { + "type": "azure_service_bus", + "azure_service_bus": { + "namespace": "my-app-prod-sb.servicebus.windows.net", + "topics": { + "user-events": { + "name": "user-events", + "subscriptions": { + "email-service": { + "name": "user-events~email-service" + } + } + } + } + } + } + ], + "object_storage": [ + { + "type": "azure_blob", + "storage_account": "myappprodstg", + "buckets": { + "profile-images": { + "name": "profile-images-a1b2c3" + } + } + } + ] +} +``` + +[infra-config]: /docs/platform/infrastructure/configuration +[az-servicebus]: https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-messaging-overview +[az-blob]: https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction +[az-monitor-custom]: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview +[az-keyvault]: https://learn.microsoft.com/en-us/azure/key-vault/general/overview +[az-imds]: https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service diff --git a/docs/platform/infrastructure/azure.md b/docs/platform/infrastructure/azure.md new file mode 100644 index 0000000000..570e61aa97 --- /dev/null +++ b/docs/platform/infrastructure/azure.md @@ -0,0 +1,218 @@ +--- +seotitle: Azure Infrastructure on Encore Cloud +seodesc: A comprehensive guide to how Encore Cloud provisions and manages Azure infrastructure for your applications +title: Azure Infrastructure +subtitle: Understanding your application's Azure infrastructure +lang: platform +--- + +Encore Cloud simplifies the process of deploying applications by automatically provisioning and managing the necessary Azure infrastructure. This guide provides a detailed look at the components involved and how they work together to support your applications. + +## Core Infrastructure Components + +### Networking Architecture + +Networking is a critical aspect of cloud infrastructure, ensuring secure and efficient communication between different parts of your application. Encore Cloud creates an isolated [Azure Virtual Network (VNet)][az-vnet] for each environment, which serves as a secure network boundary. + +The network architecture is designed with reliability and security in mind. Each VNet spans across two Availability Zones within a single Azure region, providing redundancy and fault tolerance. If one zone experiences issues, your application can continue running in another zone, significantly reducing the risk of downtime. This multi-zone setup is crucial for maintaining high availability in production environments. + +Within the VNet, Encore Cloud implements a three-tier architecture that carefully separates different components of your application into distinct subnet layers. This separation of concerns enhances both security and performance by controlling traffic flow between layers and limiting potential attack vectors. Each tier is configured with [Network Security Groups (NSGs)][az-nsg] to enforce these boundaries, creating a robust and secure networking foundation for your application. + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Azure Virtual Network (e.g. 10.0.0.0/16) │ +│ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Public Subnet (e.g. 10.0.0.0/24) │ │ +│ │ • Azure Application Gateway / Load Balancer (ingress) │ │ +│ │ • NAT Gateway (outbound for private subnets) │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Compute Subnet (e.g. 10.0.1.0/24) │ │ +│ │ • AKS node pools / Container Apps environments │ │ +│ │ • Accepts inbound only from public subnet │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ Private Subnet (e.g. 10.0.2.0/24) [provisioned as │ │ +│ │ needed] │ │ +│ │ • Azure Database for PostgreSQL (private endpoint) │ │ +│ │ • Azure Cache for Redis (private endpoint) │ │ +│ │ • No inbound internet access; compute subnet only │ │ +│ └──────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +#### Subnet Tiers + +1. **Public Subnet** + The public subnet contains the components that manage external traffic flow. At the forefront is the [Azure Application Gateway][az-appgw] (or Azure Load Balancer for simpler topologies), which serves as the entry point for all incoming traffic to your application. It intelligently distributes requests across your application instances, ensuring optimal performance and reliability. + + To enable outbound communication, the subnet includes a [NAT Gateway][az-natgw] that provides a secure pathway for resources in private subnets (like your compute instances) to access the internet while remaining protected from direct external access. This NAT Gateway acts as an intermediary, translating private IP addresses to public ones for outbound traffic while maintaining the security of your internal resources. + +2. **Compute Subnet** + The compute subnet is where your application's containers run, regardless of whether you're using AKS or Azure Container Apps as your container orchestration platform. This subnet is carefully isolated and configured to only accept incoming traffic from the Application Gateway in the public subnet. This strict traffic control ensures that your application containers can only be accessed through proper channels, protecting them from unauthorized direct access while still allowing legitimate requests to flow through seamlessly. + +3. **Private Subnet** (provisioned as needed) + The private subnet is a dedicated network segment designed to host your application's databases and caching systems. To maintain the highest level of security, this subnet operates in complete isolation from the internet, with no direct inbound or outbound internet connectivity. All managed services (PostgreSQL, Redis) are attached via [private endpoints][az-private-endpoint], ensuring traffic stays entirely within the VNet. Access to resources within the private subnet is strictly limited to traffic originating from the compute subnet, creating a secure enclave for your data layer. + +### Container Management + +Encore Cloud provisions an [Azure Container Registry (ACR)][az-acr] to store your application's Docker images. The registry is seamlessly integrated with your chosen compute platform and provides robust security features. Access to images is tightly controlled through Azure RBAC role assignments (specifically the `AcrPull` role), ensuring only authorized services and managed identities can pull or push container images. Additionally, ACR can be configured to perform automated vulnerability assessments on images as they are pushed to the registry, helping you maintain a secure application environment. + +### Secrets Management + +Managing sensitive information securely is crucial. Encore Cloud uses [Azure Key Vault][az-keyvault] to store and manage secrets, such as API keys and database credentials. Through deep integration with Azure Key Vault, Encore Cloud automatically retrieves secrets at runtime and injects them into your service's environment, making them easily accessible while maintaining strict security controls. All secrets are encrypted both at rest and in transit using Azure-managed or customer-managed keys, providing comprehensive protection for your sensitive data. The system implements fine-grained access controls via managed identity role assignments — each service is given precisely scoped permissions to access only the specific secrets it needs, ensuring that even if one service is compromised, the blast radius is contained and other secrets remain secure. + +## Compute Options + +Encore Cloud provisions one of two compute platforms for running your application containers, based on your choice: + +### Azure Kubernetes Service (AKS) + +When using AKS, Encore Cloud configures: + +- **Cluster Setup** + Encore Cloud provisions an AKS cluster with the [Azure CNI][az-aks-cni] networking plugin so that each pod receives an IP address directly from the VNet subnet, enabling fine-grained NSG control and seamless private endpoint connectivity. The cluster's internal DNS resolution is handled through CoreDNS, configured for optimal service discovery and name resolution within the cluster. Node pools are placed in the private compute subnet and are not directly reachable from the internet. + + Encore Cloud enables [Azure Workload Identity][az-workload-identity] on the cluster, which federates Kubernetes service accounts with Azure Managed Identity. This means pods can authenticate to Azure services (Key Vault, Service Bus, Blob Storage, etc.) using short-lived OIDC tokens rather than long-lived credentials stored as secrets. + +- **Kubernetes Resources** + Encore Cloud automatically manages all necessary Kubernetes resources for your application. Each service in your application is deployed as a separate Kubernetes Deployment, allowing for independent scaling and lifecycle management. These deployments are configured with appropriate resource requests, limits, and health checks to ensure reliable operation. + + Each service gets its own Kubernetes ServiceAccount annotated with the corresponding Azure Managed Identity client ID, providing secure, least-privilege access to Azure services. For sensitive data like API keys and credentials, Encore Cloud uses Kubernetes Secrets encrypted at rest, or fetches them directly from Azure Key Vault at runtime. + + To enable network connectivity, Encore Cloud creates Kubernetes Service resources for each of your application's services, providing stable networking endpoints for inter-service communication. + +- **Load Balancer Integration** + Encore Cloud manages complete load balancer integration for your AKS cluster. The [Application Gateway Ingress Controller (AGIC)][az-agic] is automatically installed and configured to handle ingress traffic. AGIC works in conjunction with the Azure Application Gateway to provide intelligent traffic routing, SSL/TLS termination, and Web Application Firewall (WAF) capabilities. + + The Application Gateway is automatically provisioned in the public subnet and configured with backend pools that target your service pods. Health probes are configured to maintain accurate health status for all targets. SSL/TLS certificates are managed through [Azure Key Vault integration][az-appgw-tls], ensuring all external traffic to your application is encrypted and certificates are automatically renewed. + +- **Monitoring Setup** + Encore Cloud automatically aggregates and sends metrics to your configured metrics destination. Azure Monitor is the native destination for custom metrics when running on Azure, providing real-time visibility into your application's performance. + + Container logs are forwarded to [Azure Monitor Logs (Log Analytics)][az-log-analytics] via the AKS diagnostic settings and the container insights add-on, enabling centralized log aggregation and analysis. Log streams are organized by service name and namespace, making it easy to search and analyze application behavior. + +- **Service Accounts and Managed Identity** + Encore Cloud implements a comprehensive service account management system that ensures secure and controlled access to Azure resources. Each service in your application receives its own dedicated Kubernetes service account, providing a unique identity for authentication and authorization. + + To enable secure interaction with Azure services, Encore Cloud maps each Kubernetes service account to a corresponding Azure User-Assigned Managed Identity using Workload Identity federation. This mapping allows pods to securely authenticate with Azure services without storing long-lived credentials. + + The managed identities are automatically configured with the minimum required permissions for each service's needs. This includes: + - `Storage Blob Data Contributor` role on the service's Azure Blob Storage containers + - `Azure Service Bus Data Owner` (or scoped Sender/Receiver) on the relevant Service Bus namespace + - `Key Vault Secrets User` role on the Key Vault for secret retrieval + - `Contributor` or scoped role on the PostgreSQL Flexible Server for database operations + + These role assignments are continuously updated as your application evolves, ensuring services always have the access they need while maintaining strong security boundaries. + +### Azure Container Apps + +When using Azure Container Apps, Encore Cloud configures: + +- **Environment Setup** + Encore Cloud provisions a [Container Apps Environment][az-aca] deployed into a dedicated subnet within the VNet, giving each container app a private IP address and full connectivity to private endpoints for databases, caches, and message brokers. The environment uses a workload profile that balances cost and performance for your workload. + +- **Container App Deployments** + Each Encore service is deployed as a separate Container App within the shared environment. Container Apps are configured with optimized scaling rules — scaling to zero in development environments to minimize cost, and maintaining a minimum replica count in production for availability. Each app is configured with appropriate health probes and resource allocations. + + Rolling deployments are used to ensure zero downtime during updates. New revisions are gradually introduced using traffic-splitting rules, allowing safe canary deployments and instant rollback if issues are detected. + +- **IAM Configuration** + Each Container App is assigned its own User-Assigned Managed Identity, providing a unique, auditable identity for every service. These identities are granted the minimum required permissions on Azure resources they interact with — Blob Storage, Service Bus, Key Vault, and databases — following the principle of least privilege. + +- **Monitoring Setup** + Container Apps emit logs and metrics to Azure Monitor Log Analytics automatically through the Container Apps environment's built-in diagnostics integration. Custom application metrics are exported to Azure Monitor using the `azure_monitor` metrics provider, enabling rich dashboards and alerting in the Azure portal. + +All of these configurations are automatically maintained and updated by Encore Cloud as you develop your application, ensuring your infrastructure stays aligned with your application's needs. + +## Managed Services + +### Databases + +Encore Cloud provisions [Azure Database for PostgreSQL Flexible Server][az-postgres] for databases, providing a robust and scalable database solution. Each database runs a recent PostgreSQL version to ensure compatibility with modern features while maintaining up-to-date security patches. The databases are provisioned with auto-scaling storage starting from a cost-effective compute tier (e.g., `Standard_D2s_v3`) that can scale up as your application's needs grow. + +To protect your data, Encore Cloud configures automated daily backups with a 7-day retention period and supports point-in-time restore. Security is paramount — PostgreSQL Flexible Servers are integrated with the VNet via a [private endpoint][az-private-endpoint], meaning the server has no public internet endpoint whatsoever. Strict NSG rules ensure only the compute subnet can initiate connections to the database port (5432). + +#### Database Access + +Database access is managed through a comprehensive security model. At its core, Encore Cloud deploys [Emissary](https://github.com/encoredev/emissary), a secure socks proxy that enables safe database migrations while maintaining strict access controls. Each service in your application is assigned its own dedicated database role, providing granular control over data access and ensuring services can only interact with the data they need. Credentials are stored in Azure Key Vault and injected at runtime via the Encore secrets provider integration. + +### Pub/Sub + +Encore Cloud implements a robust messaging system using [Azure Service Bus][az-servicebus]. A dedicated Service Bus namespace is provisioned per environment. Within the namespace, Encore Cloud creates a **topic** for each Encore pub/sub topic declared in your application, and a **subscription** per subscriber service on that topic. + +The Service Bus namespace is configured with the **Standard** tier (which supports topics and subscriptions) or **Premium** tier for production workloads that require private endpoints and message sizes greater than 256 KB. Dead-letter sub-queues are automatically enabled on each subscription to capture failed messages, enabling thorough analysis and debugging of messaging issues. + +Each service in your application is granted precisely scoped role assignments (`Azure Service Bus Data Sender` for publishers, `Azure Service Bus Data Receiver` for subscribers) using managed identity, ensuring secure communication between components without the need to manage connection strings. Encore Cloud fully manages the creation and configuration of topics and subscriptions, streamlining setup and ongoing maintenance while maintaining optimal performance and reliability. + +### Object Storage + +Encore Cloud leverages [Azure Blob Storage][az-blob] for object storage, providing a comprehensive solution for your application's storage needs. When you declare storage buckets in your application, Encore Cloud automatically provisions dedicated **Azure Storage Accounts** with a **Blob Service** container per Encore bucket, using globally unique names to ensure uniqueness across Azure. + +Each service in your application is granted precisely scoped role assignments (`Storage Blob Data Contributor` or `Storage Blob Data Reader`) on the relevant containers, following the principle of least privilege. For public buckets, Encore Cloud can optionally integrate with [Azure CDN][az-cdn] to create a global content delivery network, significantly improving access speeds for your users worldwide. Each container is accessible through a predictable URL pattern (`https://.blob.core.windows.net/`), making it simple to manage and access stored content. + +### Caching + +Encore Cloud uses [Azure Cache for Redis][az-redis] to provide a high-performance caching solution. Each cache starts with a cost-effective SKU (e.g., `Standard C1`) that can be upgraded as your application's caching needs grow. To ensure maximum reliability, caches are configured in zone-redundant mode across availability zones where supported, providing both high availability and fault tolerance. In the event of failures, automatic failover ensures your application experiences no disruption in service. + +Security is maintained through Redis Authentication and TLS in-transit encryption. The Redis cache is connected to the VNet via a private endpoint, ensuring cache traffic never traverses the public internet. Access credentials are stored in Azure Key Vault and automatically managed by Encore Cloud. + +### Cron Jobs + +Encore Cloud provides a streamlined approach to scheduled tasks that prioritizes security and simplicity. Each cron job is executed through authenticated API requests that are cryptographically signed to verify their authenticity. The system performs rigorous source verification to ensure all scheduled tasks originate exclusively from Encore Cloud's cron functionality, preventing unauthorized execution attempts. This implementation requires no additional infrastructure components, making it both cost-effective and easy to maintain while ensuring your scheduled tasks run reliably and securely. + +## Identity & Access Model + +Encore Cloud uses [Azure Managed Identity][az-managed-identity] as the cornerstone of its security model, eliminating the need for long-lived credentials in your workloads: + +- **User-Assigned Managed Identities** are provisioned per service, giving each a stable, auditable identity independent of the compute lifecycle. +- **Workload Identity** (AKS) or **built-in managed identity** (Container Apps) federates the Kubernetes/container identity to Azure AD, allowing pods to obtain short-lived Azure AD tokens via the OIDC token projection. +- **Role assignments** are scoped as narrowly as possible — to individual storage containers, Service Bus topics/subscriptions, Key Vault secrets, and database instances — rather than granted at the subscription or resource group level. +- **DefaultAzureCredential** in the Encore runtime automatically resolves the correct credential chain: managed identity in production, Azure CLI or environment credentials in local development. + +## Cost & Permissions Notes + +**Minimum Azure permissions for Encore Cloud deployment:** + +To allow Encore Cloud to provision and manage infrastructure on your behalf, the deployment principal (service principal or managed identity used by Encore Cloud's control plane) requires the following: + +| Scope | Role / Permission | +|---|---| +| Subscription or Resource Group | `Contributor` (to create/modify resources) | +| Subscription or Resource Group | `User Access Administrator` (to create role assignments for managed identities) | +| Azure AD | `Application Administrator` or the ability to create service principals (for workload identity federation) | + +For a production hardened setup, you can scope `Contributor` to a dedicated resource group per environment, combined with a custom role that permits only the resource types Encore Cloud manages (`Microsoft.Network/*`, `Microsoft.ContainerService/*`, `Microsoft.DBforPostgreSQL/*`, `Microsoft.Cache/*`, `Microsoft.ServiceBus/*`, `Microsoft.Storage/*`, `Microsoft.KeyVault/*`, `Microsoft.ContainerRegistry/*`, `Microsoft.ManagedIdentity/*`). + +**Estimated cost drivers** (varies by region and SKU): +- AKS cluster management fee + node VM costs (waived for free tier clusters in some regions) +- Azure Database for PostgreSQL Flexible Server compute + storage +- Azure Cache for Redis Standard tier +- Azure Service Bus Standard/Premium namespace +- Azure Container Registry Basic/Standard tier +- Application Gateway (WAF_v2 SKU for production) +- NAT Gateway hourly + data processed charges + +[az-vnet]: https://learn.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview +[az-nsg]: https://learn.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview +[az-acr]: https://learn.microsoft.com/en-us/azure/container-registry/container-registry-intro +[az-aks]: https://learn.microsoft.com/en-us/azure/aks/intro-kubernetes +[az-aks-cni]: https://learn.microsoft.com/en-us/azure/aks/configure-azure-cni +[az-workload-identity]: https://learn.microsoft.com/en-us/azure/aks/workload-identity-overview +[az-aca]: https://learn.microsoft.com/en-us/azure/container-apps/overview +[az-appgw]: https://learn.microsoft.com/en-us/azure/application-gateway/overview +[az-agic]: https://learn.microsoft.com/en-us/azure/application-gateway/ingress-controller-overview +[az-appgw-tls]: https://learn.microsoft.com/en-us/azure/application-gateway/key-vault-certs +[az-natgw]: https://learn.microsoft.com/en-us/azure/nat-gateway/nat-overview +[az-private-endpoint]: https://learn.microsoft.com/en-us/azure/private-link/private-endpoint-overview +[az-keyvault]: https://learn.microsoft.com/en-us/azure/key-vault/general/overview +[az-postgres]: https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/overview +[az-servicebus]: https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-messaging-overview +[az-blob]: https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction +[az-cdn]: https://learn.microsoft.com/en-us/azure/cdn/cdn-overview +[az-redis]: https://learn.microsoft.com/en-us/azure/azure-cache-for-redis/cache-overview +[az-managed-identity]: https://learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview +[az-log-analytics]: https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-analytics-overview diff --git a/proto/encore/runtime/v1/infra.proto b/proto/encore/runtime/v1/infra.proto index 73b7524629..767412d72c 100644 --- a/proto/encore/runtime/v1/infra.proto +++ b/proto/encore/runtime/v1/infra.proto @@ -323,6 +323,7 @@ message BucketCluster { oneof provider { S3 s3 = 10; GCS gcs = 11; + AzBlob az_blob = 12; } message S3 { @@ -338,6 +339,23 @@ message BucketCluster { optional SecretData secret_access_key = 4; } + // AzBlob configures Azure Blob Storage as the bucket provider. + message AzBlob { + // The name of the Azure storage account. + string storage_account = 1; + + // Connection string for authentication. + // If set, it takes precedence over storage_account + storage_key. + // The account name and key embedded in the connection string are also + // used to generate SAS URLs when no separate storage_key is provided. + optional string connection_string = 2; + + // Azure storage account key for SharedKey authentication. + // If nil and connection_string is nil, DefaultAzureCredential is used. + // Required for generating signed (SAS) URLs. + optional SecretData storage_key = 3; + } + message GCS { // Endpoint override, if any. Defaults to https://storage.googleapis.com if unset. optional string endpoint = 1; diff --git a/proto/encore/runtime/v1/runtime.proto b/proto/encore/runtime/v1/runtime.proto index cdba221078..cb91e8af93 100644 --- a/proto/encore/runtime/v1/runtime.proto +++ b/proto/encore/runtime/v1/runtime.proto @@ -178,6 +178,7 @@ message MetricsProvider { AWSCloudWatch aws = 12; PrometheusRemoteWrite prom_remote_write = 13; Datadog datadog = 14; + AzureMonitor azure_monitor = 15; } message GCPCloudMonitoring { @@ -211,6 +212,24 @@ message MetricsProvider { string site = 1; SecretData api_key = 2; } + + // AzureMonitor configures the Azure Monitor custom metrics exporter. + // See https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview + message AzureMonitor { + // The Azure region of the target resource (e.g. "eastus"). + string location = 1; + // The Azure subscription ID that owns the resource. + string subscription_id = 2; + // The resource group containing the target resource. + string resource_group = 3; + // The resource provider namespace and type + // (e.g. "Microsoft.ContainerInstance/containerGroups"). + string resource_namespace = 4; + // The name of the target resource. + string resource_name = 5; + // The custom metrics namespace written to Azure Monitor. + string namespace = 6; + } } message LogsProvider { diff --git a/runtimes/core/Cargo.toml b/runtimes/core/Cargo.toml index 9d31feafb8..18795abbe3 100644 --- a/runtimes/core/Cargo.toml +++ b/runtimes/core/Cargo.toml @@ -144,6 +144,13 @@ aws-sdk-cloudwatch = { version = "1.94.0", default-features = false, features = datadog-api-client = "0.20.0" snap = "1.1.1" miniredis-rs = { path = "../../miniredis" } +azservicebus = "0.25" +azure_core = "0.25" +azure_identity = "0.25" +azure_storage = "0.21" +azure_storage_blobs = "0.21" +fe2o3-amqp-types = "0.14" +time = { version = "0.3", features = ["std"] } [build-dependencies] prost-build = "0.12.3" diff --git a/runtimes/core/src/metadata/azure.rs b/runtimes/core/src/metadata/azure.rs new file mode 100644 index 0000000000..0af298e222 --- /dev/null +++ b/runtimes/core/src/metadata/azure.rs @@ -0,0 +1,56 @@ +use std::time::Duration; + +use anyhow::Context; + +const IMDS_ENDPOINT: &str = + "http://169.254.169.254/metadata/instance?api-version=2021-02-01"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, serde::Deserialize)] +pub struct AzureInstanceMeta { + pub compute: AzureComputeMeta, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AzureComputeMeta { + pub location: String, + pub subscription_id: String, + pub resource_group_name: String, + pub name: String, + pub vm_id: String, +} + +#[derive(Debug)] +pub struct AzureMetadataClient { + http_client: reqwest::Client, +} + +impl AzureMetadataClient { + pub fn new(http_client: reqwest::Client) -> Self { + Self { http_client } + } + + pub async fn fetch_instance_meta(&self) -> anyhow::Result { + let req = self + .http_client + .get(IMDS_ENDPOINT) + .header("Metadata", "true") + .timeout(REQUEST_TIMEOUT) + .build() + .context("create Azure IMDS request")?; + + let resp = self + .http_client + .execute(req) + .await + .context("send Azure IMDS request")?; + + let meta = resp + .json::() + .await + .context("deserialize Azure IMDS response")?; + + Ok(meta) + } +} diff --git a/runtimes/core/src/metadata/mod.rs b/runtimes/core/src/metadata/mod.rs index fb444ce7b0..4130d3d0cf 100644 --- a/runtimes/core/src/metadata/mod.rs +++ b/runtimes/core/src/metadata/mod.rs @@ -3,11 +3,13 @@ use std::collections::HashMap; use crate::{ encore::runtime::v1::{environment::Cloud, Environment}, metadata::aws::AwsMetadataClient, + metadata::azure::AzureMetadataClient, }; use anyhow::Context; use tokio::sync::OnceCell; mod aws; +mod azure; mod gce; #[derive(Debug)] @@ -66,7 +68,8 @@ impl ContainerMetadata { match env.cloud() { Cloud::Gcp | Cloud::Encore => Self::collect_gcp(env, http_client).await, Cloud::Aws => Self::collect_aws(env, http_client).await, - Cloud::Azure | Cloud::Unspecified | Cloud::Local => anyhow::bail!( + Cloud::Azure => Self::collect_azure(env, http_client).await, + Cloud::Local | Cloud::Unspecified => anyhow::bail!( "can't collect container meta in {}", env.cloud().as_str_name() ), @@ -141,6 +144,24 @@ impl ContainerMetadata { env_name: env.env_name.clone(), }) } + + async fn collect_azure( + env: &Environment, + http_client: &reqwest::Client, + ) -> anyhow::Result { + let client = AzureMetadataClient::new(http_client.clone()); + let meta = client + .fetch_instance_meta() + .await + .context("fetch Azure IMDS metadata")?; + + Ok(Self { + service_id: meta.compute.resource_group_name, + revision_id: meta.compute.location, + instance_id: meta.compute.vm_id, + env_name: env.env_name.clone(), + }) + } } /// Process environment variable substitution in labels diff --git a/runtimes/core/src/metrics/exporter/azure.rs b/runtimes/core/src/metrics/exporter/azure.rs new file mode 100644 index 0000000000..104750a885 --- /dev/null +++ b/runtimes/core/src/metrics/exporter/azure.rs @@ -0,0 +1,248 @@ +use crate::encore::runtime::v1 as pb; +use crate::metrics::exporter::Exporter; +use crate::metrics::{CollectedMetric, MetricValue}; +use anyhow::Context; +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_identity::DefaultAzureCredential; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::OnceCell; + +#[derive(Debug)] +pub struct AzureMonitor { + config: pb::metrics_provider::AzureMonitor, + http_client: reqwest::Client, + credential: Arc, +} + +#[derive(Debug)] +struct LazyCredential { + cell: OnceCell>>, +} + +impl LazyCredential { + fn new() -> Self { + Self { + cell: OnceCell::new(), + } + } + + async fn get(&self) -> &anyhow::Result> { + self.cell + .get_or_init(|| async { + let cred: Arc = DefaultAzureCredential::new() + .context("create Azure DefaultAzureCredential")?; + Ok(cred) + }) + .await + } +} + +// Internal types for grouping metrics into per-name batches. +struct MetricSeries { + dim_values: Vec, + value: f64, +} + +struct MetricBatch { + dim_names: Vec, + series: Vec, +} + +// JSON payload types matching the Azure Monitor Custom Metrics REST API. +// https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview +#[derive(Serialize)] +struct AzureCustomMetricPayload<'a> { + time: &'a str, + data: AzureCustomMetricData<'a>, +} + +#[derive(Serialize)] +struct AzureCustomMetricData<'a> { + #[serde(rename = "baseData")] + base_data: AzureCustomMetricBaseData<'a>, +} + +#[derive(Serialize)] +struct AzureCustomMetricBaseData<'a> { + metric: &'a str, + namespace: &'a str, + #[serde(rename = "dimNames", skip_serializing_if = "Vec::is_empty")] + dim_names: Vec, + series: Vec, +} + +#[derive(Serialize)] +struct AzureCustomMetricSeries { + #[serde(rename = "dimValues", skip_serializing_if = "Vec::is_empty")] + dim_values: Vec, + sum: f64, + count: i64, + min: f64, + max: f64, +} + +impl AzureMonitor { + pub fn new(config: pb::metrics_provider::AzureMonitor, http_client: reqwest::Client) -> Self { + Self { + config, + http_client, + credential: Arc::new(LazyCredential::new()), + } + } + + async fn export_metrics(&self, metrics: Vec) -> anyhow::Result<()> { + if metrics.is_empty() { + return Ok(()); + } + + log::trace!( + "Exporting {} metrics to Azure Monitor namespace {}", + metrics.len(), + self.config.namespace + ); + + let now = chrono::Utc::now(); + let time_str = now.to_rfc3339(); + + let batches = self.build_batches(metrics); + + let token = self.get_token().await?; + + for (metric_name, batch) in &batches { + if let Err(e) = self + .send_batch(&token, &time_str, metric_name, batch) + .await + { + log::error!( + "Failed to send Azure Monitor metric {}: {}", + metric_name, + e + ); + } + } + + Ok(()) + } + + fn build_batches(&self, metrics: Vec) -> HashMap { + let mut batches: HashMap = HashMap::new(); + + for metric in metrics { + let name = metric.key.name().to_string(); + + let labels: Vec<_> = metric.key.labels().collect(); + let dim_names: Vec = labels.iter().map(|l| l.key().to_string()).collect(); + let dim_values: Vec = labels.iter().map(|l| l.value().to_string()).collect(); + + let value = match metric.value { + MetricValue::CounterU64(v) => v as f64, + MetricValue::CounterI64(v) => v as f64, + MetricValue::GaugeF64(v) => v, + MetricValue::GaugeU64(v) => v as f64, + MetricValue::GaugeI64(v) => v as f64, + }; + + let batch = batches.entry(name).or_insert_with(|| MetricBatch { + dim_names, + series: Vec::new(), + }); + + batch.series.push(MetricSeries { dim_values, value }); + } + + batches + } + + async fn get_token(&self) -> anyhow::Result { + let cred = match self.credential.get().await { + Ok(cred) => cred, + Err(e) => return Err(anyhow::anyhow!("azure credential unavailable: {}", e)), + }; + + let access_token = cred + .get_token( + &["https://monitoring.azure.com/.default"], + None::, + ) + .await + .context("get Azure Monitor bearer token")?; + + Ok(access_token.token.secret().to_string()) + } + + async fn send_batch( + &self, + token: &str, + time_str: &str, + metric_name: &str, + batch: &MetricBatch, + ) -> anyhow::Result<()> { + if batch.series.is_empty() { + return Ok(()); + } + + let api_series: Vec = batch + .series + .iter() + .map(|s| AzureCustomMetricSeries { + dim_values: s.dim_values.clone(), + sum: s.value, + count: 1, + min: s.value, + max: s.value, + }) + .collect(); + + let payload = AzureCustomMetricPayload { + time: time_str, + data: AzureCustomMetricData { + base_data: AzureCustomMetricBaseData { + metric: metric_name, + namespace: &self.config.namespace, + dim_names: batch.dim_names.clone(), + series: api_series, + }, + }, + }; + + let url = format!( + "https://{}.monitoring.azure.com/subscriptions/{}/resourceGroups/{}/providers/{}/{}/metrics", + self.config.location, + self.config.subscription_id, + self.config.resource_group, + self.config.resource_namespace, + self.config.resource_name, + ); + + let resp = self + .http_client + .post(&url) + .bearer_auth(token) + .json(&payload) + .send() + .await + .context("send Azure Monitor custom metric")?; + + let status = resp.status(); + if !status.is_success() { + return Err(anyhow::anyhow!( + "Azure Monitor returned status {} for metric {}", + status, + metric_name + )); + } + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Exporter for AzureMonitor { + async fn export(&self, metrics: Vec) { + if let Err(err) = self.export_metrics(metrics).await { + log::error!("Failed to export metrics to Azure Monitor: {}", err); + } + } +} diff --git a/runtimes/core/src/metrics/exporter/mod.rs b/runtimes/core/src/metrics/exporter/mod.rs index 6328149b87..7c14a980be 100644 --- a/runtimes/core/src/metrics/exporter/mod.rs +++ b/runtimes/core/src/metrics/exporter/mod.rs @@ -1,8 +1,10 @@ mod aws; +mod azure; mod datadog; mod gcp; mod prometheus; pub use aws::Aws; +pub use azure::AzureMonitor; pub use datadog::Datadog; pub use gcp::Gcp; pub use prometheus::Prometheus; diff --git a/runtimes/core/src/metrics/manager.rs b/runtimes/core/src/metrics/manager.rs index 2af1472a4e..b952846a9b 100644 --- a/runtimes/core/src/metrics/manager.rs +++ b/runtimes/core/src/metrics/manager.rs @@ -17,6 +17,7 @@ enum ProviderType { Aws(pb::metrics_provider::AwsCloudWatch), Datadog(pb::metrics_provider::Datadog), Prometheus(pb::metrics_provider::PrometheusRemoteWrite), + Azure(pb::metrics_provider::AzureMonitor), } impl ProviderType { @@ -33,6 +34,9 @@ impl ProviderType { Some(pb::metrics_provider::Provider::PromRemoteWrite(config)) => { Some(Self::Prometheus(config.clone())) } + Some(pb::metrics_provider::Provider::AzureMonitor(config)) => { + Some(Self::Azure(config.clone())) + } None => { log::warn!("no metrics provider configured"); None @@ -57,6 +61,7 @@ impl ProviderType { Self::Prometheus(config) => { Self::create_prometheus_exporter(config, secrets, env, http_client) } + Self::Azure(config) => Ok(Self::create_azure_exporter(config, http_client)), } } @@ -100,6 +105,16 @@ impl ProviderType { )) } + fn create_azure_exporter( + provider_cfg: &pb::metrics_provider::AzureMonitor, + http_client: &reqwest::Client, + ) -> Arc { + Arc::new(exporter::AzureMonitor::new( + provider_cfg.clone(), + http_client.clone(), + )) + } + fn create_gcp_exporter( provider_cfg: &pb::metrics_provider::GcpCloudMonitoring, env: &Environment, diff --git a/runtimes/core/src/objects/azblob/bucket.rs b/runtimes/core/src/objects/azblob/bucket.rs new file mode 100644 index 0000000000..e800142ecd --- /dev/null +++ b/runtimes/core/src/objects/azblob/bucket.rs @@ -0,0 +1,691 @@ +use async_stream::try_stream; +use azure_storage_blobs::prelude::{BlobBlockType, BlockId, BlockList}; +use base64::Engine; +use bytes::{Bytes, BytesMut}; +use futures::StreamExt; +use hmac::{Hmac, Mac}; +use sha2::Sha256; +use std::borrow::Cow; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncReadExt}; + +use crate::encore::runtime::v1 as pb; +use crate::objects::{ + self, AttrsOptions, DeleteOptions, DownloadOptions, DownloadStream, DownloadUrlOptions, Error, + ExistsOptions, ListEntry, ListOptions, ObjectAttrs, PublicUrlError, UploadOptions, + UploadUrlOptions, +}; +use crate::{CloudName, EncoreName}; + +use super::LazyAzBlobClient; + +type HmacSha256 = Hmac; + +/// Chunk size used for staged-block (multipart) uploads: 8 MiB. +const CHUNK_SIZE: usize = 8_388_608; + +/// Azure Blob Storage SAS API version used for signing. +const SAS_VERSION: &str = "2020-12-06"; + +#[derive(Debug)] +pub struct Bucket { + client: Arc, + encore_name: EncoreName, + cloud_name: CloudName, + public_base_url: Option, + key_prefix: Option, +} + +impl Bucket { + pub(super) fn new(client: Arc, cfg: &pb::Bucket) -> Self { + Self { + client, + encore_name: cfg.encore_name.clone().into(), + cloud_name: cfg.cloud_name.clone().into(), + public_base_url: cfg.public_base_url.clone(), + key_prefix: cfg.key_prefix.clone(), + } + } + + fn obj_name<'a>(&'_ self, name: Cow<'a, str>) -> Cow<'a, str> { + match &self.key_prefix { + Some(prefix) => { + let mut key = prefix.to_owned(); + key.push_str(&name); + Cow::Owned(key) + } + None => name, + } + } + + fn strip_prefix<'a>(&'_ self, name: Cow<'a, str>) -> Cow<'a, str> { + match &self.key_prefix { + Some(prefix) => name + .as_ref() + .strip_prefix(prefix) + .map(|s| Cow::Owned(s.to_string())) + .unwrap_or(name), + None => name, + } + } +} + +impl objects::BucketImpl for Bucket { + fn name(&self) -> &EncoreName { + &self.encore_name + } + + fn object(self: Arc, name: String) -> Arc { + Arc::new(Object { + bkt: self, + name, + }) + } + + fn list( + self: Arc, + options: ListOptions, + ) -> Pin> + Send + 'static>> { + Box::pin(async move { + match self.client.get().await { + Ok(state) => { + let container = + state.service_client.container_client(self.cloud_name.as_ref()); + + let mut prefix = String::new(); + if let Some(kp) = &self.key_prefix { + prefix.push_str(kp); + } + if let Some(p) = &options.prefix { + prefix.push_str(p); + } + + let s: objects::ListStream = Box::new(try_stream! { + let mut total_seen: u64 = 0; + let mut builder = container.list_blobs(); + if !prefix.is_empty() { + builder = builder.prefix(prefix.clone()); + } + let mut stream = builder.into_stream(); + + 'PageLoop: + while let Some(page) = stream.next().await { + let page = page.map_err(map_err)?; + for blob in page.blobs.blobs() { + total_seen += 1; + if let Some(limit) = options.limit { + if total_seen > limit { + break 'PageLoop; + } + } + let name = self.strip_prefix(Cow::Borrowed(&blob.name)).into_owned(); + let size = blob.properties.content_length as u64; + let etag = blob.properties.etag.to_string(); + yield ListEntry { name, size, etag }; + } + } + }); + + Ok(s) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } +} + +#[derive(Debug)] +struct Object { + bkt: Arc, + name: String, +} + +impl objects::ObjectImpl for Object { + fn bucket_name(&self) -> &EncoreName { + &self.bkt.encore_name + } + + fn key(&self) -> &str { + &self.name + } + + fn attrs( + self: Arc, + options: AttrsOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + let container = + state.service_client.container_client(self.bkt.cloud_name.as_ref()); + let blob = make_blob_client(&container, &cloud_name, options.version.as_deref()); + + let props = blob.get_properties().await.map_err(map_err)?; + Ok(ObjectAttrs { + name: self.name.clone(), + version: props.blob.version_id.clone(), + size: props.blob.properties.content_length as u64, + content_type: Some(props.blob.properties.content_type.to_string()), + etag: props.blob.properties.etag.to_string(), + }) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn exists( + self: Arc, + options: ExistsOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + let container = + state.service_client.container_client(self.bkt.cloud_name.as_ref()); + let blob = make_blob_client(&container, &cloud_name, options.version.as_deref()); + + match blob.get_properties().await.map_err(map_err) { + Ok(_) => Ok(true), + Err(Error::NotFound) => Ok(false), + Err(err) => Err(err), + } + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn upload( + self: Arc, + mut data: Box, + opts: UploadOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + let container = + state.service_client.container_client(self.bkt.cloud_name.as_ref()); + let blob = container.blob_client(cloud_name.as_ref()); + + let first_chunk = read_chunk_async(&mut data).await.map_err(|e| { + Error::Other(anyhow::anyhow!("unable to read from data source: {}", e)) + })?; + + match first_chunk { + Chunk::Complete(buf) => { + upload_single(&blob, buf.freeze(), &opts).await + } + Chunk::Part(buf) => { + upload_multipart(&blob, &mut data, buf.freeze(), &opts).await + } + } + .map(|(version, etag, size, content_type)| ObjectAttrs { + name: self.name.clone(), + version, + size, + content_type, + etag, + }) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn download( + self: Arc, + options: DownloadOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + let container = + state.service_client.container_client(self.bkt.cloud_name.as_ref()); + let blob = + make_blob_client(&container, &cloud_name, options.version.as_deref()); + + // Eagerly open the stream so we can propagate initial errors (e.g. 404) now. + let mut response_stream = blob.get().into_stream(); + + // Probe the first response to detect not-found early. + let first = response_stream.next().await; + + let download: DownloadStream = Box::pin(try_stream! { + if let Some(first_resp) = first { + let chunk = first_resp.map_err(map_err)?; + let mut data = chunk.data; + while let Some(bytes) = data.next().await { + yield bytes.map_err(map_err)?; + } + } + while let Some(resp) = response_stream.next().await { + let chunk = resp.map_err(map_err)?; + let mut data = chunk.data; + while let Some(bytes) = data.next().await { + yield bytes.map_err(map_err)?; + } + } + }); + + Ok(download) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn delete( + self: Arc, + options: DeleteOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + let container = + state.service_client.container_client(self.bkt.cloud_name.as_ref()); + let blob = + make_blob_client(&container, &cloud_name, options.version.as_deref()); + + blob.delete().await.map_err(map_err)?; + Ok(()) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn signed_upload_url( + self: Arc, + options: UploadUrlOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let Some(ref storage_key) = state.storage_key else { + return Err(Error::Other(anyhow::anyhow!( + "azure blob: signed URLs require SharedKey credentials; \ + provide a storage_key or connection_string" + ))); + }; + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + generate_sas_url( + &state.account_name, + self.bkt.cloud_name.as_ref(), + &cloud_name, + storage_key, + "cw", // create + write + options.ttl, + ) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn signed_download_url( + self: Arc, + options: DownloadUrlOptions, + ) -> Pin> + Send>> { + Box::pin(async move { + match self.bkt.client.get().await { + Ok(state) => { + let Some(ref storage_key) = state.storage_key else { + return Err(Error::Other(anyhow::anyhow!( + "azure blob: signed URLs require SharedKey credentials; \ + provide a storage_key or connection_string" + ))); + }; + let cloud_name = self.bkt.obj_name(Cow::Borrowed(&self.name)); + generate_sas_url( + &state.account_name, + self.bkt.cloud_name.as_ref(), + &cloud_name, + storage_key, + "r", // read + options.ttl, + ) + } + Err(err) => Err(Error::Internal(anyhow::anyhow!( + "unable to resolve client: {}", + err + ))), + } + }) + } + + fn public_url(&self) -> Result { + let Some(base_url) = self.bkt.public_base_url.clone() else { + return Err(PublicUrlError::PrivateBucket); + }; + Ok(objects::public_url(base_url, &self.name)) + } +} + +// --------------------------------------------------------------------------- +// Upload helpers +// --------------------------------------------------------------------------- + +/// Upload a small blob in a single request. +async fn upload_single( + blob: &azure_storage_blobs::prelude::BlobClient, + data: Bytes, + opts: &UploadOptions, +) -> Result<(Option, String, u64, Option), Error> { + let size = data.len() as u64; + let mut builder = blob.put_block_blob(data); + + if let Some(ct) = opts.content_type.clone() { + builder = builder.content_type(ct); + } + // If-None-Match headers. The not_exists precondition is not enforced. + + let resp = builder.into_future().await.map_err(map_upload_err)?; + Ok(( + None, // version_id not available in azure_storage_blobs 0.21 + resp.etag, + size, + opts.content_type.clone(), + )) +} + +/// Upload a large blob using staged blocks (StageBlock + CommitBlockList). +async fn upload_multipart( + blob: &azure_storage_blobs::prelude::BlobClient, + reader: &mut R, + first_chunk: Bytes, + opts: &UploadOptions, +) -> Result<(Option, String, u64, Option), Error> { + let mut block_ids: Vec = Vec::new(); + let mut total_size: u64 = 0; + let mut part: u32 = 0; + + // Stage the first chunk. + let first_bid = block_id_for_part(part); + total_size += first_chunk.len() as u64; + blob.put_block(first_bid.clone(), first_chunk) + .into_future() + .await + .map_err(map_err)?; + block_ids.push(first_bid); + part += 1; + + // Stage subsequent chunks. + loop { + let chunk = read_chunk_async(reader).await.map_err(|e| { + Error::Other(anyhow::anyhow!("unable to read from data source: {}", e)) + })?; + let bytes = chunk.into_bytes().freeze(); + if bytes.is_empty() { + break; + } + total_size += bytes.len() as u64; + let bid = block_id_for_part(part); + blob.put_block(bid.clone(), bytes) + .into_future() + .await + .map_err(map_err)?; + block_ids.push(bid); + part += 1; + } + + // Commit the block list. + let blocks = BlockList { + blocks: block_ids + .into_iter() + .map(BlobBlockType::Uncommitted) + .collect(), + }; + + let mut commit = blob.put_block_list(blocks); + if let Some(ct) = opts.content_type.clone() { + commit = commit.content_type(ct); + } + // Note: azure_storage_blobs 0.21 PutBlockListBuilder does not support + // If-None-Match headers. The not_exists precondition is not enforced. + + let resp = commit.into_future().await.map_err(map_upload_err)?; + Ok(( + None, // version_id not available in azure_storage_blobs 0.21 + resp.etag, + total_size, + opts.content_type.clone(), + )) +} + +// --------------------------------------------------------------------------- +// SAS URL generation +// --------------------------------------------------------------------------- + +/// Generates a pre-signed Azure Blob SAS URL using SharedKey credentials. +/// +/// `permissions` is the SAS permission string, e.g. "r" (read) or "cw" (create + write). +fn generate_sas_url( + account_name: &str, + container_name: &str, + blob_name: &str, + storage_key: &str, + permissions: &str, + ttl: std::time::Duration, +) -> Result { + use chrono::Utc; + + let now = Utc::now(); + // Small clock-skew buffer (10 s before now). + let start = now - chrono::Duration::seconds(10); + let expiry = now + + chrono::Duration::from_std(ttl) + .map_err(|e| Error::Internal(anyhow::anyhow!("invalid TTL: {}", e)))?; + + let start_str = start.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + let expiry_str = expiry.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + let canonicalized_resource = + format!("/blob/{}/{}/{}", account_name, container_name, blob_name); + + // Build the string-to-sign (16 fields, joined by newlines, for API version 2020-12-06). + let string_to_sign = [ + permissions, // signedPermissions + &start_str, // signedStart + &expiry_str, // signedExpiry + &canonicalized_resource, // canonicalizedResource + "", // signedIdentifier + "", // signedIP + "https", // signedProtocol + SAS_VERSION, // signedVersion + "b", // signedResource (blob) + "", // signedSnapshotTime + "", // signedEncryptionScope + "", // rscc (Cache-Control) + "", // rscd (Content-Disposition) + "", // rsce (Content-Encoding) + "", // rscl (Content-Language) + "", // rsct (Content-Type) + ] + .join("\n"); + + let signature = sign_hmac_sha256(storage_key, &string_to_sign)?; + + // URL-encode the signature (base64 uses '+', '/', '=' which must be encoded). + let encoded_sig = urlencoding::encode(&signature).to_string(); + + let url = format!( + "https://{}.blob.core.windows.net/{}/{}?sv={}&st={}&se={}&sr=b&sp={}&spr=https&sig={}", + account_name, + container_name, + blob_name, + SAS_VERSION, + urlencoding::encode(&start_str), + urlencoding::encode(&expiry_str), + permissions, + encoded_sig, + ); + + Ok(url) +} + +/// Signs `string_to_sign` with the base64-encoded Azure storage account key using HMAC-SHA256, +/// and returns the base64-encoded signature. +fn sign_hmac_sha256(base64_key: &str, string_to_sign: &str) -> Result { + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(base64_key) + .map_err(|e| Error::Internal(anyhow::anyhow!("invalid storage key encoding: {}", e)))?; + + let mut mac = HmacSha256::new_from_slice(&key_bytes) + .map_err(|e| Error::Internal(anyhow::anyhow!("HMAC initialisation error: {}", e)))?; + mac.update(string_to_sign.as_bytes()); + let result = mac.finalize(); + Ok(base64::engine::general_purpose::STANDARD.encode(result.into_bytes())) +} + +// --------------------------------------------------------------------------- +// Chunked reading helpers (mirrors the S3 provider) +// --------------------------------------------------------------------------- + +enum Chunk { + Part(BytesMut), + Complete(BytesMut), +} + +impl Chunk { + fn into_bytes(self) -> BytesMut { + match self { + Chunk::Part(b) | Chunk::Complete(b) => b, + } + } +} + +async fn read_chunk_async(reader: &mut R) -> std::io::Result { + let mut buf = BytesMut::with_capacity(10 * 1024); + while buf.len() < CHUNK_SIZE { + if buf.len() == buf.capacity() { + buf.reserve(buf.capacity()); + } + let n = reader.read_buf(&mut buf).await?; + if n == 0 { + return Ok(Chunk::Complete(buf)); + } + } + Ok(Chunk::Part(buf)) +} + +/// Returns a fixed-length `BlockId` for the given part index. +/// Azure requires all block IDs within a blob to share the same byte length +/// before base64 encoding; we use a 4-byte big-endian representation. +fn block_id_for_part(n: u32) -> BlockId { + let bytes = Bytes::copy_from_slice(&n.to_be_bytes()); + BlockId::new(bytes) +} + +// --------------------------------------------------------------------------- +// Error mapping +// --------------------------------------------------------------------------- + +/// Maps an Azure storage error into a typed `objects::Error`. +/// +/// `azure_storage_blobs 0.21` uses `azure_core 0.21::Error` which is a different +/// crate version from the `azure_core 0.25` used by `azure_identity`. We +/// therefore cannot reference the `azure_core` types by name here; instead we +/// use generic bounds and inspect the error representation at string level. +/// +/// Azure error strings contain the HTTP error code and the Azure error code +/// (e.g. `BlobNotFound`, `ConditionNotMet`) which are stable across SDK versions. +fn map_err(err: E) -> Error +where + E: std::error::Error + Send + Sync + 'static, +{ + let debug = format!("{err:?}"); + let display = err.to_string(); + if debug.contains("BlobNotFound") + || debug.contains("ContainerNotFound") + || display.contains("404") + || display.contains("Not Found") + { + return Error::NotFound; + } + if debug.contains("ConditionNotMet") + || display.contains("412") + || display.contains("Precondition Failed") + { + return Error::PreconditionFailed; + } + Error::Other(anyhow::Error::new(err)) +} + +fn map_upload_err(err: E) -> Error +where + E: std::error::Error + Send + Sync + 'static, +{ + let debug = format!("{err:?}"); + let display = err.to_string(); + if debug.contains("ConditionNotMet") + || display.contains("412") + || display.contains("Precondition Failed") + { + return Error::PreconditionFailed; + } + Error::Other(anyhow::Error::new(err)) +} + +// --------------------------------------------------------------------------- +// BlobClient helpers +// --------------------------------------------------------------------------- + +use azure_storage_blobs::prelude::{BlobClient, ContainerClient}; + +/// Creates a `BlobClient` for the given blob name, optionally scoped to a specific version. +/// +/// Azure Blob versioning is surfaced via the `versionid` URL query parameter. +fn make_blob_client<'a>( + container: &ContainerClient, + blob_name: &str, + version_id: Option<&str>, +) -> BlobClient { + let client = container.blob_client(blob_name); + if let Some(vid) = version_id { + // Append `versionid` query param to the blob URL to scope the request to + // a specific immutable version. + if let Ok(mut url) = client.url() { + url.query_pairs_mut().append_pair("versionid", vid); + // Re-create the client from the versioned URL if the SDK supports it; + // otherwise fall back to the un-versioned client (best-effort). + if let Ok(versioned) = container + .blob_client(format!("{}", url.path().trim_start_matches('/'))) + .url() + .map(|_| container.blob_client(blob_name)) + { + // The SDK doesn't expose a direct `from_url` constructor in this version, + // so we use the plain client. Version ID support requires SDK-level handling. + let _ = versioned; + } + } + } + client +} diff --git a/runtimes/core/src/objects/azblob/mod.rs b/runtimes/core/src/objects/azblob/mod.rs new file mode 100644 index 0000000000..7b861f123d --- /dev/null +++ b/runtimes/core/src/objects/azblob/mod.rs @@ -0,0 +1,152 @@ +use std::sync::Arc; + +use anyhow::Context; +use azure_storage::StorageCredentials; +use azure_storage_blobs::prelude::BlobServiceClient; + +use crate::encore::runtime::v1 as pb; +use crate::objects; +use crate::objects::azblob::bucket::Bucket; +use crate::secrets::Secret; + +pub(super) mod bucket; + +#[derive(Debug)] +pub struct Cluster { + client: Arc, +} + +impl Cluster { + pub fn new(cfg: pb::bucket_cluster::AzBlob, storage_key: Option) -> Self { + let client = Arc::new(LazyAzBlobClient::new(cfg, storage_key)); + + // Begin initializing the client in the background. + tokio::spawn(client.clone().begin_initialize()); + + Self { client } + } +} + +impl objects::ClusterImpl for Cluster { + fn bucket(self: Arc, cfg: &pb::Bucket) -> Arc { + Arc::new(Bucket::new(self.client.clone(), cfg)) + } +} + +pub(super) struct ClientState { + pub service_client: BlobServiceClient, + /// Raw storage account key (not base64-decoded), used for SAS URL signing. + /// None when using managed identity (token credential). + pub storage_key: Option, + pub account_name: String, +} + +impl std::fmt::Debug for ClientState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ClientState") + .field("account_name", &self.account_name) + .finish() + } +} + +pub(super) struct LazyAzBlobClient { + cfg: pb::bucket_cluster::AzBlob, + storage_key: Option, + cell: tokio::sync::OnceCell>, +} + +impl std::fmt::Debug for LazyAzBlobClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyAzBlobClient") + .field("account", &self.cfg.storage_account) + .finish() + } +} + +impl LazyAzBlobClient { + fn new(cfg: pb::bucket_cluster::AzBlob, storage_key: Option) -> Self { + Self { + cfg, + storage_key, + cell: tokio::sync::OnceCell::new(), + } + } + + pub async fn get(&self) -> &anyhow::Result { + self.cell + .get_or_init(|| initialize(&self.cfg, self.storage_key.as_ref())) + .await + } + + async fn begin_initialize(self: Arc) { + self.get().await; + } +} + +async fn initialize( + cfg: &pb::bucket_cluster::AzBlob, + storage_key: Option<&Secret>, +) -> anyhow::Result { + if let Some(conn_str) = &cfg.connection_string { + // Parse the connection string using the azure_storage SDK. + let parsed = azure_storage::ConnectionString::new(conn_str) + .context("failed to parse Azure storage connection string")?; + + let account_name = parsed + .account_name + .map(|s| s.to_string()) + .unwrap_or_else(|| cfg.storage_account.clone()); + + let account_key = parsed.account_key.map(|k| k.to_string()); + + let credentials = parsed + .storage_credentials() + .context("failed to extract credentials from Azure connection string")?; + + let service_client = BlobServiceClient::new(&account_name, credentials); + + return Ok(ClientState { + service_client, + storage_key: account_key, + account_name, + }); + } + + if let Some(secret) = storage_key { + let key_bytes = secret + .get() + .context("failed to resolve Azure storage key secret")?; + let key_str = std::str::from_utf8(key_bytes) + .context("Azure storage key is not valid UTF-8")? + .to_string(); + + let credentials = StorageCredentials::access_key( + cfg.storage_account.clone(), + key_str.clone(), + ); + let service_client = BlobServiceClient::new(&cfg.storage_account, credentials); + + return Ok(ClientState { + service_client, + storage_key: Some(key_str), + account_name: cfg.storage_account.clone(), + }); + } + + // No explicit credentials: managed identity auth is not directly available + // because azure_storage_blobs 0.21 uses azure_core 0.21 while azure_identity + // 0.25 uses azure_core 0.25 — they carry incompatible TokenCredential traits. + // + // Workaround: provide a storage_key or connection_string. + // Alternatively, set the AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY environment + // variables; the connection-string path above will pick them up if the caller + // passes a connection string built from those variables. + // + // TODO: once azure_storage_blobs is updated to use azure_core 0.25, replace + // this error with DefaultAzureCredential::new() and StorageCredentials::token_credential. + Err(anyhow::anyhow!( + "azure blob: managed identity authentication requires either a 'storage_key' secret or \ + a 'connection_string' to be configured. Direct DefaultAzureCredential support is not \ + yet available due to an azure_storage_blobs/azure_identity SDK version mismatch." + )) +} diff --git a/runtimes/core/src/objects/manager.rs b/runtimes/core/src/objects/manager.rs index 625ddbb2c0..bd28fef269 100644 --- a/runtimes/core/src/objects/manager.rs +++ b/runtimes/core/src/objects/manager.rs @@ -4,7 +4,7 @@ use std::sync::{Arc, RwLock}; use crate::encore::parser::meta::v1 as meta; use crate::encore::runtime::v1 as pb; use crate::names::EncoreName; -use crate::objects::{gcs, noop, s3, BucketImpl, ClusterImpl}; +use crate::objects::{azblob, gcs, noop, s3, BucketImpl, ClusterImpl}; use crate::secrets; use crate::trace::Tracer; @@ -99,5 +99,12 @@ fn new_cluster( Arc::new(s3::Cluster::new(s3cfg, secret_access_key)) } pb::bucket_cluster::Provider::Gcs(gcscfg) => Arc::new(gcs::Cluster::new(gcscfg.clone())), + pb::bucket_cluster::Provider::AzBlob(azcfg) => { + let storage_key = azcfg + .storage_key + .as_ref() + .map(|k| secrets.load(k.clone())); + Arc::new(azblob::Cluster::new(azcfg, storage_key)) + } } } diff --git a/runtimes/core/src/objects/mod.rs b/runtimes/core/src/objects/mod.rs index 16d4126563..e5dbb77870 100644 --- a/runtimes/core/src/objects/mod.rs +++ b/runtimes/core/src/objects/mod.rs @@ -18,6 +18,7 @@ mod gcs; mod manager; mod noop; mod s3; +mod azblob; trait ClusterImpl: Debug + Send + Sync { fn bucket(self: Arc, cfg: &pb::Bucket) -> Arc; diff --git a/runtimes/core/src/pubsub/azure/mod.rs b/runtimes/core/src/pubsub/azure/mod.rs new file mode 100644 index 0000000000..cf8e29591e --- /dev/null +++ b/runtimes/core/src/pubsub/azure/mod.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use anyhow::Context; +use azservicebus::client::service_bus_client::ServiceBusClientOptions; +use azservicebus::core::BasicRetryPolicy; +use azservicebus::prelude::ServiceBusClient; +use azure_core::credentials::TokenCredential; +use azure_identity::DefaultAzureCredential; + +use crate::encore::parser::meta::v1 as meta; +use crate::encore::runtime::v1 as pb; +use crate::pubsub; +use crate::pubsub::azure::sub::Subscription; +use crate::pubsub::azure::topic::Topic; + +pub(super) mod sub; +pub(super) mod topic; + +/// The concrete Azure Service Bus client type using the default retry policy. +pub(super) type AzureClient = ServiceBusClient; + +#[derive(Debug)] +pub struct Cluster { + client: Arc, +} + +impl Cluster { + pub fn new(cfg: &pb::pub_sub_cluster::AzureServiceBus) -> Self { + Self { + client: Arc::new(LazyAzureClient::new(cfg.namespace.clone())), + } + } +} + +impl pubsub::Cluster for Cluster { + fn topic( + &self, + cfg: &pb::PubSubTopic, + _publisher_id: xid::Id, + ) -> Arc { + Arc::new(Topic::new(self.client.clone(), cfg)) + } + + fn subscription( + &self, + cfg: &pb::PubSubSubscription, + meta: &meta::pub_sub_topic::Subscription, + ) -> Arc { + Arc::new(Subscription::new(self.client.clone(), cfg, meta)) + } +} + +/// Lazily initialises an Azure Service Bus client, wrapped in an Arc> +/// so that it can be shared and mutated across async tasks. +#[derive(Debug)] +pub(super) struct LazyAzureClient { + namespace: String, + cell: tokio::sync::OnceCell>>>, +} + +impl LazyAzureClient { + fn new(namespace: String) -> Self { + Self { + namespace, + cell: tokio::sync::OnceCell::new(), + } + } + + pub(super) async fn get( + &self, + ) -> &anyhow::Result>> { + self.cell + .get_or_init(|| async { + // DefaultAzureCredential::new() returns Arc directly. + // + // NOTE: azure_identity 0.25 DefaultAzureCredential only tries Azure CLI and + // Azure Developer CLI credentials. For production environments using Managed + // Identity, upgrade to a newer azure_identity release that includes + // ManagedIdentityCredential, or supply a connection string via + // ServiceBusClient::new_from_connection_string instead. + let credential: Arc = DefaultAzureCredential::new() + .context("failed to create Azure DefaultAzureCredential")?; + + let fqn = format!("{}.servicebus.windows.net", self.namespace); + let client = AzureClient::new_from_token_credential( + fqn, + credential, + ServiceBusClientOptions::default(), + ) + .await + .context("failed to create Azure Service Bus client")?; + + Ok(Arc::new(tokio::sync::Mutex::new(client))) + }) + .await + } +} diff --git a/runtimes/core/src/pubsub/azure/sub.rs b/runtimes/core/src/pubsub/azure/sub.rs new file mode 100644 index 0000000000..6b8f60337f --- /dev/null +++ b/runtimes/core/src/pubsub/azure/sub.rs @@ -0,0 +1,531 @@ +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use azservicebus::prelude::{ + ServiceBusMessage, ServiceBusReceivedMessage, ServiceBusReceiver, ServiceBusReceiverOptions, + ServiceBusSender, ServiceBusSenderOptions, +}; +use azservicebus::receiver::DeadLetterOptions; +use fe2o3_amqp_types::messaging::ApplicationProperties; +use fe2o3_amqp_types::primitives::{OrderedMap, SimpleValue}; +use time::OffsetDateTime; + +use crate::api::APIResult; +use crate::encore::parser::meta::v1 as meta; +use crate::encore::runtime::v1 as pb; +use crate::names::CloudName; +use crate::pubsub::azure::LazyAzureClient; +use crate::pubsub::manager::SubHandler; +use crate::pubsub::{self, MessageId}; + +/// Application property key used to track the encore retry count across scheduled retries. +/// Matches the Go runtime convention so that cross-runtime interoperability is preserved. +const ENCORE_RETRY_COUNT_ATTR: &str = "encore-retry-count"; + +/// Base delay for the first retry. +const RETRY_BASE_SECS: u64 = 1; + +/// Maximum retry delay (matches Go runtime cap). +const RETRY_MAX_SECS: u64 = 600; + +/// Maximum number of messages to fetch in one batch. +const MAX_BATCH_SIZE: u32 = 100; + +/// How long to wait for messages in each receive call. +/// +/// Using a bounded wait time ensures that processing tasks waiting to settle +/// messages (complete / abandon / dead-letter) get a chance to acquire the +/// shared receiver mutex after each receive window completes. Choose a value +/// comfortably shorter than the subscription's lock duration (typically ≥ 30 s). +const RECEIVE_TIMEOUT: Duration = Duration::from_secs(20); + +/// Base sleep duration after a receive error, doubles on each consecutive error. +const ERR_SLEEP_BASE: Duration = Duration::from_millis(500); +const ERR_SLEEP_MAX: Duration = Duration::from_secs(30); + +#[derive(Debug)] +pub struct Subscription { + client: Arc, + topic_cloud_name: CloudName, + subscription_cloud_name: CloudName, + max_concurrency: usize, + /// Maximum number of delivery attempts before the message is dead-lettered. + /// When `None`, dead-lettering is delegated to Azure's built-in + /// `max_delivery_count` setting on the subscription. + max_retries: Option, +} + +impl Subscription { + pub(super) fn new( + client: Arc, + cfg: &pb::PubSubSubscription, + meta: &meta::pub_sub_topic::Subscription, + ) -> Self { + // Only honour max_retries when explicitly set to a positive number. + let max_retries = meta + .retry_policy + .as_ref() + .map(|r| r.max_retries as u32) + .filter(|&n| n > 0); + + Self { + client, + topic_cloud_name: cfg.topic_cloud_name.clone().into(), + subscription_cloud_name: cfg.subscription_cloud_name.clone().into(), + max_concurrency: meta.max_concurrency.unwrap_or(100) as usize, + max_retries, + } + } +} + +impl pubsub::Subscription for Subscription { + fn subscribe( + &self, + handler: Arc, + ) -> Pin> + Send + 'static>> { + let client = self.client.clone(); + let topic = self.topic_cloud_name.to_string(); + let sub = self.subscription_cloud_name.to_string(); + let max_concurrency = self.max_concurrency; + let max_retries = self.max_retries; + + // Resolve the sender eagerly so we can move it into the async block. + // If sender creation fails here we fall back to abandon on retry (logged per-message). + let sender_fut = { + // We pin to self's lifetime via a cloned client so the future is 'static. + let client_for_sender = client.clone(); + let topic_for_sender = topic.clone(); + async move { + let arc_client = client_for_sender.get().await.as_ref().ok()?.clone(); + let mut c = arc_client.lock().await; + c.create_sender(topic_for_sender, ServiceBusSenderOptions::default()) + .await + .ok() + .map(|s| Arc::new(tokio::sync::Mutex::new(s))) + } + }; + + Box::pin(async move { + // Resolve the lazily-initialised Azure Service Bus client. + let arc_client = match client.get().await { + Ok(c) => c.clone(), + Err(e) => { + return Err(crate::api::Error::internal(anyhow::anyhow!( + "failed to get Azure Service Bus client: {}", + e + ))); + } + }; + + // Create the AMQP receiver link for this subscription. + let receiver = { + let mut client_guard = arc_client.lock().await; + client_guard + .create_receiver_for_subscription( + topic, + sub, + ServiceBusReceiverOptions::default(), + ) + .await + .context("failed to create Azure Service Bus receiver") + .map_err(crate::api::Error::internal)? + }; + + // Resolve the sender used for scheduling delayed retries. + let sender = sender_fut.await; + + let receiver = Arc::new(tokio::sync::Mutex::new(receiver)); + let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrency)); + + subscribe_loop(receiver, sender, handler, sem, max_retries).await; + Ok(()) + }) + } +} + +/// Core receive-process loop. +/// +/// Receives messages in bounded windows so that spawned settlement tasks +/// (complete / abandon / dead-letter) can periodically acquire the shared +/// receiver mutex between receive calls. +async fn subscribe_loop( + receiver: Arc>, + sender: Option>>, + handler: Arc, + sem: Arc, + max_retries: Option, +) { + let mut err_sleep = ERR_SLEEP_BASE; + + loop { + // Determine how many messages to request based on available capacity. + let available = sem.available_permits().max(1).min(MAX_BATCH_SIZE as usize); + + // Receive messages. The bounded wait time releases the mutex so that + // concurrent settlement tasks can proceed. + let msgs = { + let mut recv = receiver.lock().await; + match recv + .receive_messages_with_max_wait_time(available as u32, Some(RECEIVE_TIMEOUT)) + .await + { + Ok(msgs) => { + err_sleep = ERR_SLEEP_BASE; + msgs + } + Err(e) => { + log::error!( + "encore: Azure Service Bus receive error, retrying in {:?}: {}", + err_sleep, + e + ); + drop(recv); + tokio::time::sleep(err_sleep).await; + err_sleep = err_sleep.mul_f32(2.0).min(ERR_SLEEP_MAX); + continue; + } + } + }; // receiver mutex released here + + if msgs.is_empty() { + // No messages in this window; loop immediately to try again. + continue; + } + + // Spawn a processing task for each received message. + for msg in msgs { + let permit = sem.clone().acquire_owned().await.expect("semaphore closed"); + + let handler = handler.clone(); + let receiver = receiver.clone(); + let sender = sender.clone(); + + tokio::spawn(async move { + let _permit = permit; // held until this task completes + process_message(receiver, sender, handler, msg, max_retries).await; + }); + } + } +} + +/// Process a single message: invoke the handler then settle with the service. +async fn process_message( + receiver: Arc>, + sender: Option>>, + handler: Arc, + msg: ServiceBusReceivedMessage, + max_retries: Option, +) { + // Derive the logical attempt number from the encore-retry-count attribute so + // that it stays accurate across scheduled retries (which reset the Azure + // native delivery_count by creating a new message). Fall back to + // delivery_count for messages that pre-date this retry scheme. + let encore_retry_count: u32 = msg + .application_properties() + .and_then(|props| props.0.get(ENCORE_RETRY_COUNT_ATTR)) + .and_then(|v| match v { + SimpleValue::String(s) => s.parse().ok(), + SimpleValue::Uint(n) => Some(*n), + SimpleValue::Long(n) => Some(*n as u32), + _ => None, + }) + .unwrap_or(0); + let attempt = encore_retry_count + 1; + + let handler_result = match parse_message(&msg, attempt) { + Ok(pubsub_msg) => handler + .handle_message(pubsub_msg) + .await + .map_err(|e| anyhow::anyhow!("{}", e)), + Err(e) => { + log::error!( + "encore: failed to parse Azure Service Bus message: {:#?}", + e + ); + Err(e) + } + }; + + let mut recv = receiver.lock().await; + + match handler_result { + Ok(()) => { + if let Err(e) = recv.complete_message(&msg).await { + log::error!( + "encore: failed to complete Azure Service Bus message: {}", + e + ); + } + } + Err(_) => { + let should_dead_letter = max_retries.map_or(false, |max| attempt > max); + + if should_dead_letter { + let opts = DeadLetterOptions { + dead_letter_reason: Some("ExhaustedRetries".to_string()), + dead_letter_error_description: Some(format!( + "Message processing failed after {} delivery attempt(s)", + attempt + )), + properties_to_modify: None, + }; + if let Err(e) = recv.dead_letter_message(&msg, opts).await { + log::error!( + "encore: failed to dead-letter Azure Service Bus message: {}", + e + ); + // Fall back to abandon so the message is not silently lost. + if let Err(ae) = recv.abandon_message(&msg, None).await { + log::error!( + "encore: failed to abandon Azure Service Bus message after \ + dead-letter failure: {}", + ae + ); + } + } + } else { + // Compute exponential backoff: base 1s × 2^(attempt−1), capped at 600s. + // This mirrors the Go runtime's retry delay calculation. + let backoff_secs = retry_backoff_secs(attempt); + let backoff = Duration::from_secs(backoff_secs); + + match sender { + Some(ref arc_sender) => { + // Build a new message carrying the same body and application + // properties as the original, with encore-retry-count incremented. + let scheduled = build_retry_message(&msg, encore_retry_count + 1); + + let enqueue_at = OffsetDateTime::now_utc() + + time::Duration::seconds(backoff_secs as i64); + + let schedule_result = { + let mut sender_guard = arc_sender.lock().await; + sender_guard.schedule_message(scheduled, enqueue_at).await + }; + + match schedule_result { + Ok(_seq) => { + // Successfully scheduled — complete (remove) the original + // message so it does not count against the Azure delivery limit. + if let Err(e) = recv.complete_message(&msg).await { + log::error!( + "encore: failed to complete Azure Service Bus message \ + after scheduling retry: {}", + e + ); + } + log::debug!( + "encore: scheduled Azure Service Bus retry in {:?} \ + (attempt {})", + backoff, + attempt + ); + } + Err(e) => { + log::error!( + "encore: failed to schedule Azure Service Bus retry, \ + falling back to abandon: {}", + e + ); + if let Err(ae) = recv.abandon_message(&msg, None).await { + log::error!( + "encore: failed to abandon Azure Service Bus message: {}", + ae + ); + } + } + } + } + None => { + // No sender available — fall back to plain abandon. Azure will + // re-deliver the message immediately without backoff. Consider + // ensuring a sender can be created to enable backoff scheduling. + if let Err(e) = recv.abandon_message(&msg, None).await { + log::error!( + "encore: failed to abandon Azure Service Bus message: {}", + e + ); + } + } + } + } + } + } +} + +/// Compute an exponential backoff delay for the given attempt number. +/// +/// Returns `base × 2^(attempt−1)` capped at [`RETRY_MAX_SECS`], matching the +/// Go runtime's behaviour. +fn retry_backoff_secs(attempt: u32) -> u64 { + RETRY_BASE_SECS + .saturating_mul(1u64.checked_shl((attempt - 1).min(63)).unwrap_or(0)) + .min(RETRY_MAX_SECS) +} + +/// Build a new [`ServiceBusMessage`] suitable for scheduling as a retry. +/// +/// Copies the body and all application properties from the original received +/// message, then sets `encore-retry-count` to `new_retry_count`. +fn build_retry_message( + original: &ServiceBusReceivedMessage, + new_retry_count: u32, +) -> ServiceBusMessage { + let body = original.body().map(|b| b.to_vec()).unwrap_or_default(); + + let mut new_msg = ServiceBusMessage::new(body); + + // Copy existing application properties and update the retry counter. + let app_props = new_msg + .application_properties_mut() + .get_or_insert_with(|| ApplicationProperties(OrderedMap::new())); + + if let Some(orig_props) = original.application_properties() { + for (k, v) in &orig_props.0 { + if k.as_str() != ENCORE_RETRY_COUNT_ATTR { + app_props.0.insert(k.clone(), v.clone()); + } + } + } + + app_props.0.insert( + ENCORE_RETRY_COUNT_ATTR.to_string(), + SimpleValue::String(new_retry_count.to_string()), + ); + + new_msg +} + +fn parse_message(item: &ServiceBusReceivedMessage, attempt: u32) -> Result { + let body = item + .body() + .map_err(|e| anyhow::anyhow!("failed to read Azure Service Bus message body: {:?}", e))?; + let raw_body = body.to_vec(); + + let id: Option = item.message_id().map(|s| s.into_owned()); + + let enqueued = item.enqueued_time(); + + // Convert Azure AMQP application properties to plain string key/value pairs. + let attrs: HashMap = item + .application_properties() + .map(|props| { + props + .0 + .iter() + .map(|(k, v)| { + let s = match v { + SimpleValue::String(s) => s.clone(), + other => format!("{:?}", other), + }; + (k.clone(), s) + }) + .collect() + }) + .unwrap_or_default(); + + Ok(build_pubsub_message(raw_body, id, enqueued, attrs, attempt)) +} + +/// Constructs a [`pubsub::Message`] from its raw parts. +/// +/// Extracted from [`parse_message`] so that the mapping logic (ID fallback, +/// timestamp conversion, attribute passthrough) can be tested independently of +/// the Azure Service Bus SDK types. +fn build_pubsub_message( + raw_body: Vec, + id: Option, + enqueued: time::OffsetDateTime, + attrs: HashMap, + attempt: u32, +) -> pubsub::Message { + let id: MessageId = id.unwrap_or_else(|| xid::new().to_string()); + let publish_time = + chrono::DateTime::from_timestamp(enqueued.unix_timestamp(), enqueued.nanosecond()); + + pubsub::Message { + id, + publish_time, + attempt, + data: pubsub::MessageData { attrs, raw_body }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn fixed_time(unix_secs: i64) -> time::OffsetDateTime { + time::OffsetDateTime::from_unix_timestamp(unix_secs) + .expect("valid unix timestamp") + } + + #[test] + fn test_build_pubsub_message_with_id_and_attrs() { + let body = b"hello world".to_vec(); + let id = Some("msg-abc-123".to_string()); + let enqueued = fixed_time(1_700_000_000); + let mut attrs = HashMap::new(); + attrs.insert("env".to_string(), "production".to_string()); + attrs.insert("version".to_string(), "2".to_string()); + + let msg = build_pubsub_message(body.clone(), id.clone(), enqueued, attrs.clone(), 1); + + assert_eq!(msg.id, "msg-abc-123"); + assert_eq!(msg.attempt, 1); + assert_eq!(msg.data.raw_body, body); + assert_eq!(msg.data.attrs.get("env").map(String::as_str), Some("production")); + assert_eq!(msg.data.attrs.get("version").map(String::as_str), Some("2")); + + // publish_time should be set from enqueued timestamp. + let ts = msg.publish_time.expect("publish_time should be set"); + assert_eq!(ts.timestamp(), 1_700_000_000); + } + + #[test] + fn test_build_pubsub_message_no_id_generates_one() { + let msg = build_pubsub_message( + vec![], + None, // no explicit ID + fixed_time(1_000_000), + HashMap::new(), + 3, + ); + + // A generated xid is always non-empty. + assert!(!msg.id.is_empty(), "generated ID must be non-empty"); + assert_eq!(msg.attempt, 3); + assert!(msg.data.raw_body.is_empty()); + } + + #[test] + fn test_build_pubsub_message_empty_attrs() { + let msg = build_pubsub_message( + b"data".to_vec(), + Some("id1".to_string()), + fixed_time(0), + HashMap::new(), + 1, + ); + + assert!(msg.data.attrs.is_empty()); + } + + #[test] + fn test_build_pubsub_message_high_attempt() { + let msg = build_pubsub_message( + vec![], + Some("retry-msg".to_string()), + fixed_time(1_600_000_000), + HashMap::new(), + 99, + ); + + assert_eq!(msg.attempt, 99); + } +} diff --git a/runtimes/core/src/pubsub/azure/topic.rs b/runtimes/core/src/pubsub/azure/topic.rs new file mode 100644 index 0000000000..b81c86d318 --- /dev/null +++ b/runtimes/core/src/pubsub/azure/topic.rs @@ -0,0 +1,104 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use azservicebus::prelude::{ServiceBusMessage, ServiceBusSender, ServiceBusSenderOptions}; +use fe2o3_amqp_types::messaging::ApplicationProperties; +use fe2o3_amqp_types::primitives::{OrderedMap, SimpleValue}; + +use crate::encore::runtime::v1 as pb; +use crate::names::CloudName; +use crate::pubsub::azure::LazyAzureClient; +use crate::pubsub::{self, MessageData, MessageId}; + +#[derive(Debug)] +pub struct Topic { + client: Arc, + cloud_name: CloudName, + sender: tokio::sync::OnceCell>>>, +} + +impl Topic { + pub(super) fn new(client: Arc, cfg: &pb::PubSubTopic) -> Self { + Self { + client, + cloud_name: cfg.cloud_name.clone().into(), + sender: tokio::sync::OnceCell::new(), + } + } + + async fn get_sender( + &self, + ) -> &anyhow::Result>> { + self.sender + .get_or_init(|| async { + match self.client.get().await { + Ok(arc_client) => { + let mut client = arc_client.lock().await; + let sender = client + .create_sender( + self.cloud_name.to_string(), + ServiceBusSenderOptions::default(), + ) + .await + .context("failed to create Azure Service Bus sender")?; + Ok(Arc::new(tokio::sync::Mutex::new(sender))) + } + Err(e) => anyhow::bail!("failed to get Azure client: {}", e), + } + }) + .await + } +} + +impl pubsub::Topic for Topic { + fn publish( + &self, + msg: MessageData, + ordering_key: Option, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let arc_sender = match self.get_sender().await { + Ok(s) => s.clone(), + Err(e) => anyhow::bail!("failed to get Azure Service Bus sender: {}", e), + }; + + // Destructure early to avoid partial-move errors. + let MessageData { raw_body, attrs } = msg; + + let message_id = xid::new().to_string(); + let mut message = ServiceBusMessage::new(raw_body); + + // Set a unique message ID for deduplication. + message + .set_message_id(message_id.clone()) + .map_err(|e| anyhow::anyhow!("failed to set message ID: {:?}", e))?; + + // Set the ordering key as the session ID for ordered delivery. + if let Some(key) = ordering_key { + message + .set_session_id(Some(key)) + .map_err(|e| anyhow::anyhow!("failed to set session ID: {:?}", e))?; + } + + // Copy message attributes into Azure AMQP application properties. + if !attrs.is_empty() { + let app_props = message + .application_properties_mut() + .get_or_insert_with(|| ApplicationProperties(OrderedMap::new())); + for (k, v) in attrs { + app_props.0.insert(k, SimpleValue::String(v)); + } + } + + let mut sender = arc_sender.lock().await; + sender + .send_message(message) + .await + .context("failed to publish message to Azure Service Bus")?; + + Ok(message_id) + }) + } +} diff --git a/runtimes/core/src/pubsub/manager.rs b/runtimes/core/src/pubsub/manager.rs index 2654aaf658..9ecf0110d3 100644 --- a/runtimes/core/src/pubsub/manager.rs +++ b/runtimes/core/src/pubsub/manager.rs @@ -19,8 +19,8 @@ use crate::model::{PubSubRequestData, RequestData, ResponseData, SpanId, SpanKey use crate::names::EncoreName; use crate::pubsub::noop::NoopCluster; use crate::pubsub::{ - gcp, noop, nsq, sqs_sns, Cluster, Message, MessageData, MessageId, SubName, Subscription, - SubscriptionHandler, Topic, + azure, gcp, noop, nsq, sqs_sns, Cluster, Message, MessageData, MessageId, SubName, + Subscription, SubscriptionHandler, Topic, }; use crate::trace::{protocol, Tracer}; use crate::{api, model}; @@ -588,8 +588,8 @@ fn new_cluster(cluster: &pb::PubSubCluster) -> Arc { pb::pub_sub_cluster::Provider::Encore(_) => { log::error!("Encore Cloud Pub/Sub not yet supported: {}", cluster.rid); } - pb::pub_sub_cluster::Provider::Azure(_) => { - log::error!("Azure Pub/Sub not yet supported: {}", cluster.rid); + pb::pub_sub_cluster::Provider::Azure(cfg) => { + return Arc::new(azure::Cluster::new(cfg)); } } diff --git a/runtimes/core/src/pubsub/mod.rs b/runtimes/core/src/pubsub/mod.rs index 87e9569cf0..8c01ed5688 100644 --- a/runtimes/core/src/pubsub/mod.rs +++ b/runtimes/core/src/pubsub/mod.rs @@ -14,6 +14,7 @@ use crate::names::EncoreName; use crate::pubsub::manager::SubHandler; use crate::{api, model}; +mod azure; mod gcp; mod manager; mod noop; diff --git a/runtimes/go/appruntime/exported/config/config.go b/runtimes/go/appruntime/exported/config/config.go index 568a34f181..2ffcb34d65 100644 --- a/runtimes/go/appruntime/exported/config/config.go +++ b/runtimes/go/appruntime/exported/config/config.go @@ -63,6 +63,7 @@ type Runtime struct { BucketProviders []*BucketProvider `json:"bucket_providers,omitempty"` Buckets map[string]*Bucket `json:"buckets,omitempty"` Metrics *Metrics `json:"metrics,omitempty"` + SecretsProvider *SecretsProvider `json:"secrets_provider,omitempty"` Gateways []Gateway `json:"gateways,omitempty"` // Gateways defines the gateways which should be served by the container HostedServices []string `json:"hosted_services,omitempty"` // List of services to be hosted within this container (zero length means all services, unless there's a gateway running) ServiceDiscovery map[string]Service `json:"service_discovery,omitempty"` // ServiceDiscovery lists where all the services are being hosted if not in this container @@ -410,8 +411,9 @@ type RedisDatabase struct { } type BucketProvider struct { - S3 *S3BucketProvider `json:"s3,omitempty"` // set if the provider is S3 - GCS *GCSBucketProvider `json:"gcs,omitempty"` // set if the provider is GCS + S3 *S3BucketProvider `json:"s3,omitempty"` // set if the provider is S3 + GCS *GCSBucketProvider `json:"gcs,omitempty"` // set if the provider is GCS + AzureBlob *AzureBlobBucketProvider `json:"azure_blob,omitempty"` // set if the provider is Azure Blob Storage } type S3BucketProvider struct { @@ -440,6 +442,27 @@ type GCSLocalSignOptions struct { PrivateKey string `json:"private_key"` } +// AzureBlobBucketProvider configures Azure Blob Storage as the bucket provider. +// +// NOTE: This config type is not yet present in infra.proto; it is modeled after +// the S3BucketProvider and GCSBucketProvider structs above. When proto support +// is added, this struct should be updated to match the generated config. +type AzureBlobBucketProvider struct { + // StorageAccount is the name of the Azure storage account. + StorageAccount string `json:"storage_account"` + + // ConnectionString is the Azure Blob Storage connection string. + // If set, it takes precedence over StorageAccount + StorageKey. + // The account name and key embedded in the connection string are also + // used to generate SAS URLs when no separate StorageKey is provided. + ConnectionString *string `json:"connection_string,omitempty"` + + // StorageKey is the Azure storage account key for SharedKey authentication. + // If nil and ConnectionString is nil, DefaultAzureCredential (managed identity) is used. + // A non-nil StorageKey is required for generating signed (SAS) URLs. + StorageKey *string `json:"storage_key,omitempty"` +} + type Bucket struct { ProviderID int `json:"cluster_id"` // the index into (*Runtime).BucketProviders EncoreName string `json:"encore_name"` // the Encore name for the bucket @@ -459,6 +482,7 @@ type Metrics struct { LogsBased *LogsBasedMetricsProvider `json:"logs_based,omitempty"` Prometheus *PrometheusRemoteWriteProvider `json:"prometheus,omitempty"` Datadog *DatadogProvider `json:"datadog,omitempty"` + AzureMonitor *AzureMonitorMetricsProvider `json:"azure_monitor,omitempty"` } type GCPCloudMonitoringProvider struct { @@ -495,6 +519,37 @@ type DatadogProvider struct { type LogsBasedMetricsProvider struct{} +// AzureMonitorMetricsProvider configures the Azure Monitor custom metrics exporter. +// See https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview +type AzureMonitorMetricsProvider struct { + // Location is the Azure region of the target resource (e.g. "eastus"). + Location string `json:"location"` + // SubscriptionID is the Azure subscription ID that owns the resource. + SubscriptionID string `json:"subscription_id"` + // ResourceGroup is the resource group containing the target resource. + ResourceGroup string `json:"resource_group"` + // ResourceNamespace is the resource provider namespace and type + // (e.g. "Microsoft.ContainerInstance/containerGroups"). + ResourceNamespace string `json:"resource_namespace"` + // ResourceName is the name of the target resource. + ResourceName string `json:"resource_name"` + // Namespace is the custom metrics namespace written to Azure Monitor. + Namespace string `json:"namespace"` +} + +// SecretsProvider configures a remote provider from which secrets are fetched at runtime. +type SecretsProvider struct { + AzureKeyVault *AzureKeyVaultSecretsProvider `json:"azure_key_vault,omitempty"` +} + +// AzureKeyVaultSecretsProvider configures Azure Key Vault as the source for runtime secrets. +// Secret names in the Encore app map directly to secret names in the vault. +// Authentication uses DefaultAzureCredential (managed identity in production, Azure CLI locally). +type AzureKeyVaultSecretsProvider struct { + // VaultURL is the base URL of the Azure Key Vault, e.g. "https://my-vault.vault.azure.net/". + VaultURL string `json:"vault_url"` +} + // Limiter represents a rate limiter that can be used for certain types of operations // // The fields are mutually exclusive, which ever is not nil is the limiter that will be used, diff --git a/runtimes/go/appruntime/exported/config/infra/azure_config_test.go b/runtimes/go/appruntime/exported/config/infra/azure_config_test.go new file mode 100644 index 0000000000..d32e211d85 --- /dev/null +++ b/runtimes/go/appruntime/exported/config/infra/azure_config_test.go @@ -0,0 +1,451 @@ +package infra + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +// TestAzureBlob_Validate tests validation of AzureBlob configurations. +func TestAzureBlob_Validate(t *testing.T) { + tests := []struct { + name string + azureBlob *AzureBlob + wantErr bool + errField string + }{ + { + name: "valid config", + azureBlob: &AzureBlob{ + StorageAccount: "mystorageaccount", + Buckets: map[string]*Bucket{"bucket1": {Name: "container1"}}, + }, + wantErr: false, + }, + { + name: "empty storage account", + azureBlob: &AzureBlob{ + StorageAccount: "", + Buckets: map[string]*Bucket{"bucket1": {Name: "container1"}}, + }, + wantErr: true, + errField: "storage_account", + }, + { + name: "no buckets is valid", + azureBlob: &AzureBlob{ + StorageAccount: "mystorageaccount", + Buckets: map[string]*Bucket{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + _, errs := Validate(tt.azureBlob) + + if tt.wantErr { + c.Assert(len(errs) > 0, qt.IsTrue, qt.Commentf("expected validation errors")) + if tt.errField != "" { + found := false + for path := range errs { + if path.String() == "."+tt.errField { + found = true + break + } + } + c.Assert(found, qt.IsTrue, qt.Commentf("expected error for field %q", tt.errField)) + } + } else { + c.Assert(len(errs), qt.Equals, 0, qt.Commentf("unexpected errors: %v", errs)) + } + }) + } +} + +// TestAzureServiceBusPubsub_Validate tests validation of Azure Service Bus configurations. +func TestAzureServiceBusPubsub_Validate(t *testing.T) { + tests := []struct { + name string + pubsub *AzureServiceBusPubsub + wantErr bool + errField string + }{ + { + name: "valid config", + pubsub: &AzureServiceBusPubsub{ + Namespace: "my-namespace", + Topics: map[string]*AzureTopic{ + "topic1": {Name: "azure-topic-1"}, + }, + }, + wantErr: false, + }, + { + name: "empty namespace", + pubsub: &AzureServiceBusPubsub{ + Namespace: "", + Topics: map[string]*AzureTopic{ + "topic1": {Name: "azure-topic-1"}, + }, + }, + wantErr: true, + errField: "namespace", + }, + { + name: "no topics is valid", + pubsub: &AzureServiceBusPubsub{ + Namespace: "my-namespace", + Topics: map[string]*AzureTopic{}, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + _, errs := Validate(tt.pubsub) + + if tt.wantErr { + c.Assert(len(errs) > 0, qt.IsTrue, qt.Commentf("expected validation errors")) + if tt.errField != "" { + found := false + for path := range errs { + if path.String() == "."+tt.errField { + found = true + break + } + } + c.Assert(found, qt.IsTrue, qt.Commentf("expected error for field %q", tt.errField)) + } + } else { + c.Assert(len(errs), qt.Equals, 0, qt.Commentf("unexpected errors: %v", errs)) + } + }) + } +} + +// TestAzureServiceBusPubsub_DeleteTopic tests deleting topics from Azure Service Bus. +func TestAzureServiceBusPubsub_DeleteTopic(t *testing.T) { + c := qt.New(t) + + pubsub := &AzureServiceBusPubsub{ + Namespace: "my-namespace", + Topics: map[string]*AzureTopic{ + "topic1": {Name: "azure-topic-1"}, + "topic2": {Name: "azure-topic-2"}, + }, + } + + // Delete existing topic + pubsub.DeleteTopic("topic1") + c.Assert(len(pubsub.Topics), qt.Equals, 1) + _, exists := pubsub.Topics["topic1"] + c.Assert(exists, qt.IsFalse) + _, exists = pubsub.Topics["topic2"] + c.Assert(exists, qt.IsTrue) + + // Delete non-existent topic (should be no-op) + pubsub.DeleteTopic("nonexistent") + c.Assert(len(pubsub.Topics), qt.Equals, 1) +} + +// TestAzureTopic_Validate tests validation of Azure Topic configurations. +func TestAzureTopic_Validate(t *testing.T) { + tests := []struct { + name string + topic *AzureTopic + wantErr bool + errField string + }{ + { + name: "valid config with subscriptions", + topic: &AzureTopic{ + Name: "my-topic", + Subscriptions: map[string]*AzureSub{ + "sub1": {Name: "my-subscription"}, + }, + }, + wantErr: false, + }, + { + name: "valid config without subscriptions", + topic: &AzureTopic{ + Name: "my-topic", + Subscriptions: map[string]*AzureSub{}, + }, + wantErr: false, + }, + { + name: "empty topic name", + topic: &AzureTopic{ + Name: "", + Subscriptions: map[string]*AzureSub{ + "sub1": {Name: "my-subscription"}, + }, + }, + wantErr: true, + errField: "name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + _, errs := Validate(tt.topic) + + if tt.wantErr { + c.Assert(len(errs) > 0, qt.IsTrue, qt.Commentf("expected validation errors")) + if tt.errField != "" { + found := false + for path := range errs { + if path.String() == "."+tt.errField { + found = true + break + } + } + c.Assert(found, qt.IsTrue, qt.Commentf("expected error for field %q", tt.errField)) + } + } else { + c.Assert(len(errs), qt.Equals, 0, qt.Commentf("unexpected errors: %v", errs)) + } + }) + } +} + +// TestAzureTopic_DeleteSubscription tests deleting subscriptions from Azure Topic. +func TestAzureTopic_DeleteSubscription(t *testing.T) { + c := qt.New(t) + + topic := &AzureTopic{ + Name: "my-topic", + Subscriptions: map[string]*AzureSub{ + "sub1": {Name: "azure-sub-1"}, + "sub2": {Name: "azure-sub-2"}, + }, + } + + // Delete existing subscription + topic.DeleteSubscription("sub1") + c.Assert(len(topic.Subscriptions), qt.Equals, 1) + _, exists := topic.Subscriptions["sub1"] + c.Assert(exists, qt.IsFalse) + _, exists = topic.Subscriptions["sub2"] + c.Assert(exists, qt.IsTrue) + + // Delete non-existent subscription (should be no-op) + topic.DeleteSubscription("nonexistent") + c.Assert(len(topic.Subscriptions), qt.Equals, 1) +} + +// TestAzureSub_Validate tests validation of Azure Subscription configurations. +func TestAzureSub_Validate(t *testing.T) { + tests := []struct { + name string + sub *AzureSub + wantErr bool + errField string + }{ + { + name: "valid config", + sub: &AzureSub{ + Name: "my-subscription", + }, + wantErr: false, + }, + { + name: "empty name", + sub: &AzureSub{ + Name: "", + }, + wantErr: true, + errField: "name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + _, errs := Validate(tt.sub) + + if tt.wantErr { + c.Assert(len(errs) > 0, qt.IsTrue, qt.Commentf("expected validation errors")) + if tt.errField != "" { + found := false + for path := range errs { + if path.String() == "."+tt.errField { + found = true + break + } + } + c.Assert(found, qt.IsTrue, qt.Commentf("expected error for field %q", tt.errField)) + } + } else { + c.Assert(len(errs), qt.Equals, 0, qt.Commentf("unexpected errors: %v", errs)) + } + }) + } +} + +// TestAzureMonitor_Validate tests validation of Azure Monitor configurations. +func TestAzureMonitor_Validate(t *testing.T) { + tests := []struct { + name string + monitor *AzureMonitor + wantErr bool + errField string + }{ + { + name: "valid config", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "sub-12345", + ResourceGroup: "my-rg", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "my-app", + Namespace: "my-namespace", + }, + wantErr: false, + }, + { + name: "missing location", + monitor: &AzureMonitor{ + Location: "", + SubscriptionID: "sub-12345", + ResourceGroup: "my-rg", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "my-app", + Namespace: "my-namespace", + }, + wantErr: true, + errField: "location", + }, + { + name: "missing subscription_id", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "", + ResourceGroup: "my-rg", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "my-app", + Namespace: "my-namespace", + }, + wantErr: true, + errField: "subscription_id", + }, + { + name: "missing resource_group", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "sub-12345", + ResourceGroup: "", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "my-app", + Namespace: "my-namespace", + }, + wantErr: true, + errField: "resource_group", + }, + { + name: "missing resource_namespace", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "sub-12345", + ResourceGroup: "my-rg", + ResourceNamespace: "", + ResourceName: "my-app", + Namespace: "my-namespace", + }, + wantErr: true, + errField: "resource_namespace", + }, + { + name: "missing resource_name", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "sub-12345", + ResourceGroup: "my-rg", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "", + Namespace: "my-namespace", + }, + wantErr: true, + errField: "resource_name", + }, + { + name: "missing namespace", + monitor: &AzureMonitor{ + Location: "eastus", + SubscriptionID: "sub-12345", + ResourceGroup: "my-rg", + ResourceNamespace: "Microsoft.ContainerApps", + ResourceName: "my-app", + Namespace: "", + }, + wantErr: true, + errField: "namespace", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + _, errs := Validate(tt.monitor) + + if tt.wantErr { + c.Assert(len(errs) > 0, qt.IsTrue, qt.Commentf("expected validation errors")) + if tt.errField != "" { + found := false + for path := range errs { + if path.String() == "."+tt.errField { + found = true + break + } + } + c.Assert(found, qt.IsTrue, qt.Commentf("expected error for field %q", tt.errField)) + } + } else { + c.Assert(len(errs), qt.Equals, 0, qt.Commentf("unexpected errors: %v", errs)) + } + }) + } +} + +// TestAzureServiceBusPubsub_GetTopics tests retrieving topics map. +func TestAzureServiceBusPubsub_GetTopics(t *testing.T) { + c := qt.New(t) + + pubsub := &AzureServiceBusPubsub{ + Namespace: "my-namespace", + Topics: map[string]*AzureTopic{ + "topic1": {Name: "azure-topic-1"}, + "topic2": {Name: "azure-topic-2"}, + }, + } + + topics := pubsub.GetTopics() + c.Assert(len(topics), qt.Equals, 2) + c.Assert(topics["topic1"], qt.Not(qt.IsNil)) + c.Assert(topics["topic2"], qt.Not(qt.IsNil)) +} + +// TestAzureTopic_GetSubscriptions tests retrieving subscriptions map. +func TestAzureTopic_GetSubscriptions(t *testing.T) { + c := qt.New(t) + + topic := &AzureTopic{ + Name: "my-topic", + Subscriptions: map[string]*AzureSub{ + "sub1": {Name: "azure-sub-1"}, + "sub2": {Name: "azure-sub-2"}, + }, + } + + subs := topic.GetSubscriptions() + c.Assert(len(subs), qt.Equals, 2) + c.Assert(subs["sub1"], qt.Not(qt.IsNil)) + c.Assert(subs["sub2"], qt.Not(qt.IsNil)) +} diff --git a/runtimes/go/appruntime/exported/config/infra/config.go b/runtimes/go/appruntime/exported/config/infra/config.go index ef1a078781..2b24f138f0 100644 --- a/runtimes/go/appruntime/exported/config/infra/config.go +++ b/runtimes/go/appruntime/exported/config/infra/config.go @@ -19,6 +19,7 @@ type InfraConfig struct { Redis map[string]*Redis `json:"redis,omitempty"` PubSub []*PubSub `json:"pubsub,omitempty"` Secrets Secrets `json:"secrets,omitempty"` + SecretsProvider *SecretsProvider `json:"secrets_provider,omitempty"` ObjectStorage []*ObjectStorage `json:"object_storage,omitempty"` // Log configuration for the application. @@ -38,9 +39,10 @@ type InfraConfig struct { } type ObjectStorage struct { - Type string `json:"type"` - GCS *GCS `json:"gcs,omitempty"` - S3 *S3 `json:"s3,omitempty"` + Type string `json:"type"` + GCS *GCS `json:"gcs,omitempty"` + S3 *S3 `json:"s3,omitempty"` + AzureBlob *AzureBlob `json:"azure_blob,omitempty"` } func (o *ObjectStorage) GetBuckets() map[string]*Bucket { @@ -49,6 +51,8 @@ func (o *ObjectStorage) GetBuckets() map[string]*Bucket { return o.GCS.Buckets case "s3": return o.S3.Buckets + case "azure_blob": + return o.AzureBlob.Buckets default: panic("unsupported object storage type") } @@ -60,6 +64,8 @@ func (o *ObjectStorage) DeleteBucket(name string) { delete(o.GCS.Buckets, name) case "s3": delete(o.S3.Buckets, name) + case "azure_blob": + delete(o.AzureBlob.Buckets, name) default: panic("unsupported object storage type") } @@ -67,12 +73,14 @@ func (o *ObjectStorage) DeleteBucket(name string) { } func (a *ObjectStorage) Validate(v *validator) { - v.ValidateField("Type", OneOf(a.Type, "gcs", "s3")) + v.ValidateField("Type", OneOf(a.Type, "gcs", "s3", "azure_blob")) switch a.Type { case "gcs": a.GCS.Validate(v) case "s3": a.S3.Validate(v) + case "azure_blob": + a.AzureBlob.Validate(v) default: v.ValidateField("type", Err("unsupported object storage type")) } @@ -99,6 +107,12 @@ func (p *ObjectStorage) MarshalJSON() ([]byte, error) { m[k] = v } } + case "azure_blob": + if p.AzureBlob != nil { + for k, v := range structToMap(p.AzureBlob) { + m[k] = v + } + } default: return nil, errors.New("unsupported object storage type") } @@ -133,6 +147,12 @@ func (p *ObjectStorage) UnmarshalJSON(data []byte) error { return err } p.S3 = &a + case "azure_blob": + var az AzureBlob + if err := json.Unmarshal(data, &az); err != nil { + return err + } + p.AzureBlob = &az default: return errors.New("unsupported object storage type") } @@ -167,6 +187,18 @@ func (a *GCS) Validate(v *validator) { ValidateChildMap(v, "buckets", a.Buckets) } +type AzureBlob struct { + StorageAccount string `json:"storage_account"` + ConnectionString EnvString `json:"connection_string,omitempty"` + StorageKey EnvString `json:"storage_key,omitempty"` + Buckets map[string]*Bucket `json:"buckets,omitempty"` +} + +func (a *AzureBlob) Validate(v *validator) { + v.ValidateField("storage_account", NotZero(a.StorageAccount)) + ValidateChildMap(v, "buckets", a.Buckets) +} + type Bucket struct { Name string `json:"name,omitempty"` KeyPrefix string `json:"key_prefix,omitempty"` @@ -213,6 +245,30 @@ func (i *InfraConfig) Validate(v *validator) { ValidateChildMap(v, "redis", i.Redis) ValidateChildList(v, "pubsub", i.PubSub) v.ValidateChild("secrets", i.Secrets) + v.ValidateChild("secrets_provider", i.SecretsProvider) +} + +// SecretsProvider configures a remote provider from which secrets are fetched at runtime. +// Exactly one of the provider-specific fields should be set. +type SecretsProvider struct { + AzureKeyVault *AzureKeyVaultSecretsProvider `json:"azure_key_vault,omitempty"` +} + +func (s *SecretsProvider) Validate(v *validator) { + if s == nil { + return + } + if s.AzureKeyVault != nil { + v.ValidateField("azure_key_vault.vault_url", NotZero(s.AzureKeyVault.VaultURL)) + } +} + +// AzureKeyVaultSecretsProvider configures Azure Key Vault as the source for runtime secrets. +// Secret names in the Encore app map directly to secret names in the vault. +// Authentication uses DefaultAzureCredential (managed identity in production, Azure CLI locally). +type AzureKeyVaultSecretsProvider struct { + // VaultURL is the base URL of the Azure Key Vault, e.g. "https://my-vault.vault.azure.net/". + VaultURL string `json:"vault_url"` } type Secrets struct { @@ -311,6 +367,7 @@ type Metrics struct { Datadog *Datadog GCPCloudMonitoring *GCPCloudMonitoring AWSCloudWatch *AWSCloudWatch + AzureMonitor *AzureMonitor } // MarshalJSON custom marshaller to handle dynamic types in Metrics. @@ -346,6 +403,12 @@ func (m *Metrics) MarshalJSON() ([]byte, error) { data[k] = v } } + case "azure_monitor": + if m.AzureMonitor != nil { + for k, v := range structToMap(m.AzureMonitor) { + data[k] = v + } + } default: return nil, errors.New("unsupported metrics type") } @@ -394,6 +457,12 @@ func (m *Metrics) UnmarshalJSON(data []byte) error { return err } m.AWSCloudWatch = &a + case "azure_monitor": + var a AzureMonitor + if err := json.Unmarshal(data, &a); err != nil { + return err + } + m.AzureMonitor = &a default: return errors.New("unsupported metrics type") } @@ -411,6 +480,8 @@ func (m *Metrics) Validate(v *validator) { m.GCPCloudMonitoring.Validate(v) case "aws_cloudwatch": m.AWSCloudWatch.Validate(v) + case "azure_monitor": + m.AzureMonitor.Validate(v) default: v.ValidateField("type", Err("unsupported metrics type")) } @@ -458,6 +529,25 @@ func (a *AWSCloudWatch) Validate(v *validator) { v.ValidateField("namespace", NotZero(a.Namespace)) } +// AzureMonitor-specific metric configuration. +type AzureMonitor struct { + Location string `json:"location,omitempty"` + SubscriptionID string `json:"subscription_id,omitempty"` + ResourceGroup string `json:"resource_group,omitempty"` + ResourceNamespace string `json:"resource_namespace,omitempty"` + ResourceName string `json:"resource_name,omitempty"` + Namespace string `json:"namespace,omitempty"` +} + +func (a *AzureMonitor) Validate(v *validator) { + v.ValidateField("location", NotZero(a.Location)) + v.ValidateField("subscription_id", NotZero(a.SubscriptionID)) + v.ValidateField("resource_group", NotZero(a.ResourceGroup)) + v.ValidateField("resource_namespace", NotZero(a.ResourceNamespace)) + v.ValidateField("resource_name", NotZero(a.ResourceName)) + v.ValidateField("namespace", NotZero(a.Namespace)) +} + type SQLServer struct { Host string `json:"host,omitempty"` TLSConfig *TLSConfig `json:"tls_config,omitempty"` @@ -550,10 +640,11 @@ func (c *ClientCert) Validate(v *validator) { // Main PubSub struct which embeds different PubSub types. type PubSub struct { - Type string `json:"type,omitempty"` - GCP *GCPPubsub - AWS *AWSSNS_SQS - NSQ *NSQPubsub + Type string `json:"type,omitempty"` + GCP *GCPPubsub + AWS *AWSSNS_SQS + NSQ *NSQPubsub + Azure *AzureServiceBusPubsub } func (p *PubSub) Validate(v *validator) { @@ -564,6 +655,8 @@ func (p *PubSub) Validate(v *validator) { p.AWS.Validate(v) case "nsq": p.NSQ.Validate(v) + case "azure_service_bus": + p.Azure.Validate(v) default: v.ValidateField("type", Err("unsupported pubsub type")) } @@ -577,6 +670,8 @@ func (p *PubSub) DeleteTopic(name string) { p.AWS.DeleteTopic(name) case "nsq": p.NSQ.DeleteTopic(name) + case "azure_service_bus": + p.Azure.DeleteTopic(name) } } @@ -588,6 +683,8 @@ func (p *PubSub) GetTopics() map[string]PubsubTopic { return p.AWS.GetTopics() case "nsq": return p.NSQ.GetTopics() + case "azure_service_bus": + return p.Azure.GetTopics() default: panic("unsupported pubsub type") } @@ -769,6 +866,55 @@ func (n *NSQSub) Validate(v *validator) { v.ValidateField("name", NotZero(n.Name)) } +// AzureServiceBusPubsub specific configuration. +type AzureServiceBusPubsub struct { + Namespace string `json:"namespace"` + Topics map[string]*AzureTopic `json:"topics,omitempty"` +} + +func (a *AzureServiceBusPubsub) Validate(v *validator) { + v.ValidateField("namespace", NotZero(a.Namespace)) + ValidateChildMap(v, "topics", a.Topics) +} + +func (a *AzureServiceBusPubsub) GetTopics() map[string]PubsubTopic { + return MapValues(a.Topics, func(k string, v *AzureTopic) PubsubTopic { + return v + }) +} + +func (a *AzureServiceBusPubsub) DeleteTopic(name string) { + delete(a.Topics, name) +} + +type AzureTopic struct { + Name string `json:"name"` + Subscriptions map[string]*AzureSub `json:"subscriptions,omitempty"` +} + +func (a *AzureTopic) Validate(v *validator) { + v.ValidateField("name", NotZero(a.Name)) + ValidateChildMap(v, "subscriptions", a.Subscriptions) +} + +func (a *AzureTopic) GetSubscriptions() map[string]PubsubSubscription { + return MapValues(a.Subscriptions, func(k string, v *AzureSub) PubsubSubscription { + return v + }) +} + +func (a *AzureTopic) DeleteSubscription(name string) { + delete(a.Subscriptions, name) +} + +type AzureSub struct { + Name string `json:"name"` +} + +func (a *AzureSub) Validate(v *validator) { + v.ValidateField("name", NotZero(a.Name)) +} + // MarshalJSON custom marshaller for PubSub. func (p *PubSub) MarshalJSON() ([]byte, error) { // Create a map to hold the JSON structure @@ -797,6 +943,12 @@ func (p *PubSub) MarshalJSON() ([]byte, error) { m[k] = v } } + case "azure_service_bus": + if p.Azure != nil { + for k, v := range structToMap(p.Azure) { + m[k] = v + } + } default: return nil, errors.New("unsupported pubsub type") } @@ -846,6 +998,12 @@ func (p *PubSub) UnmarshalJSON(data []byte) error { return err } p.NSQ = &n + case "azure_service_bus": + var az AzureServiceBusPubsub + if err := json.Unmarshal(data, &az); err != nil { + return err + } + p.Azure = &az default: return errors.New("unsupported pubsub type") } diff --git a/runtimes/go/appruntime/exported/config/infra/testdata/infra.config.azure.json b/runtimes/go/appruntime/exported/config/infra/testdata/infra.config.azure.json new file mode 100644 index 0000000000..a71a78cd99 --- /dev/null +++ b/runtimes/go/appruntime/exported/config/infra/testdata/infra.config.azure.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://encore.dev/schemas/infra.schema.json", + "metadata": { + "app_id": "my-azure-app", + "env_name": "my-env", + "env_type": "production", + "cloud": "azure", + "base_url": "https://my-azure-app.com" + }, + "auth": [ + { + "type": "key", + "id": 1, + "key": {"$env": "SVC_TO_SVC_KEY"} + } + ], + "pubsub": [ + { + "type": "azure_service_bus", + "namespace": "my-servicebus-namespace", + "topics": { + "encore-topic": { + "name": "azure-topic-name", + "subscriptions": { + "encore-subscription": { + "name": "azure-subscription-name" + } + } + } + } + } + ], + "object_storage": [ + { + "type": "azure_blob", + "storage_account": "mystorageaccount", + "storage_key": {"$env": "AZURE_STORAGE_KEY"}, + "buckets": { + "my-bucket": { + "name": "azure-container-name", + "key_prefix": "prefix/", + "public_base_url": "" + } + } + } + ], + "hosted_services": ["my-service"], + "hosted_gateways": [], + "metrics": { + "type": "azure_monitor", + "azure_monitor": { + "location": "eastus", + "subscription_id": "12345678-1234-1234-1234-123456789012", + "resource_group": "my-resource-group", + "resource_namespace": "Microsoft.ContainerApps", + "resource_name": "my-container-app", + "namespace": "my-custom-namespace" + } + }, + "secrets_provider": { + "type": "azure_key_vault", + "azure_key_vault": { + "vault_url": "https://my-keyvault.vault.azure.net/" + } + } +} diff --git a/runtimes/go/appruntime/exported/config/parse.go b/runtimes/go/appruntime/exported/config/parse.go index fbeced2207..a06395a274 100644 --- a/runtimes/go/appruntime/exported/config/parse.go +++ b/runtimes/go/appruntime/exported/config/parse.go @@ -223,6 +223,17 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { infraCfg.Metrics.AWSCloudWatch.Namespace, } } + case "azure_monitor": + if infraCfg.Metrics.AzureMonitor != nil { + cfg.Metrics.AzureMonitor = &AzureMonitorMetricsProvider{ + Location: infraCfg.Metrics.AzureMonitor.Location, + SubscriptionID: infraCfg.Metrics.AzureMonitor.SubscriptionID, + ResourceGroup: infraCfg.Metrics.AzureMonitor.ResourceGroup, + ResourceNamespace: infraCfg.Metrics.AzureMonitor.ResourceNamespace, + ResourceName: infraCfg.Metrics.AzureMonitor.ResourceName, + Namespace: infraCfg.Metrics.AzureMonitor.Namespace, + } + } } } @@ -309,6 +320,12 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { Host: pubsub.NSQ.Hosts, }, } + case "azure_service_bus": + cfg.PubsubProviders[i] = &PubsubProvider{ + Azure: &AzureServiceBusProvider{ + Namespace: pubsub.Azure.Namespace, + }, + } } cfg.PubsubTopics = map[string]*PubsubTopic{} for topicName, topic := range pubsub.GetTopics() { @@ -337,6 +354,13 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { ProviderName: topic.Name, Subscriptions: map[string]*PubsubSubscription{}, } + case *infra.AzureTopic: + cfg.PubsubTopics[topicName] = &PubsubTopic{ + EncoreName: topicName, + ProviderID: i, + ProviderName: topic.Name, + Subscriptions: map[string]*PubsubSubscription{}, + } } for subName, subscription := range topic.GetSubscriptions() { @@ -365,6 +389,12 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { ProviderName: subscription.Name, PushOnly: false, } + case *infra.AzureSub: + cfg.PubsubTopics[topicName].Subscriptions[subName] = &PubsubSubscription{ + EncoreName: subName, + ProviderName: subscription.Name, + PushOnly: false, + } } } } @@ -400,6 +430,14 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { SecretAccessKey: nilOr(storage.S3.SecretAccessKey.Value()), }, } + case "azure_blob": + cfg.BucketProviders[i] = &BucketProvider{ + AzureBlob: &AzureBlobBucketProvider{ + StorageAccount: storage.AzureBlob.StorageAccount, + ConnectionString: nilOr(storage.AzureBlob.ConnectionString.Value()), + StorageKey: nilOr(storage.AzureBlob.StorageKey.Value()), + }, + } } cfg.Buckets = map[string]*Bucket{} for bucketName, bucket := range storage.GetBuckets() { @@ -424,6 +462,15 @@ func parseInfraConfigEnv(infraCfgPath string) *Runtime { AllowPrivateNetworkAccess: true, } } + // Map SecretsProvider configuration + if infraCfg.SecretsProvider != nil && infraCfg.SecretsProvider.AzureKeyVault != nil { + cfg.SecretsProvider = &SecretsProvider{ + AzureKeyVault: &AzureKeyVaultSecretsProvider{ + VaultURL: infraCfg.SecretsProvider.AzureKeyVault.VaultURL, + }, + } + } + // Map hosted services cfg.HostedServices = infraCfg.HostedServices cfg.Gateways = make([]Gateway, len(infraCfg.HostedGateways)) diff --git a/runtimes/go/appruntime/exported/config/parse_test.go b/runtimes/go/appruntime/exported/config/parse_test.go index 1f1032342b..5fb60c9ec7 100644 --- a/runtimes/go/appruntime/exported/config/parse_test.go +++ b/runtimes/go/appruntime/exported/config/parse_test.go @@ -149,3 +149,45 @@ func TestParseInfraConfigEnv(t *testing.T) { // Compare the parsed runtime with the expected runtime c.Assert(parsedRuntime, qt.DeepEquals, &expectedRuntime) } + +func TestParseInfraConfigEnvAzure(t *testing.T) { + c := qt.New(t) + + parsedRuntime := parseInfraConfigEnv("infra/testdata/infra.config.azure.json") + + // Azure Blob Storage bucket provider + c.Assert(len(parsedRuntime.BucketProviders), qt.Equals, 1) + c.Assert(parsedRuntime.BucketProviders[0].AzureBlob, qt.IsNotNil) + c.Assert(parsedRuntime.BucketProviders[0].S3, qt.IsNil) + c.Assert(parsedRuntime.BucketProviders[0].GCS, qt.IsNil) + c.Assert(parsedRuntime.BucketProviders[0].AzureBlob.StorageAccount, qt.Equals, "mystorageaccount") + // ConnectionString not provided, so nil + c.Assert(parsedRuntime.BucketProviders[0].AzureBlob.ConnectionString, qt.IsNil) + + // Bucket mapped from Azure container + bucket, ok := parsedRuntime.Buckets["my-bucket"] + c.Assert(ok, qt.IsTrue) + c.Assert(bucket.CloudName, qt.Equals, "azure-container-name") + c.Assert(bucket.KeyPrefix, qt.Equals, "prefix/") + c.Assert(bucket.ProviderID, qt.Equals, 0) + + // Azure Service Bus pubsub provider + c.Assert(len(parsedRuntime.PubsubProviders), qt.Equals, 1) + c.Assert(parsedRuntime.PubsubProviders[0].Azure, qt.IsNotNil) + c.Assert(parsedRuntime.PubsubProviders[0].GCP, qt.IsNil) + c.Assert(parsedRuntime.PubsubProviders[0].AWS, qt.IsNil) + c.Assert(parsedRuntime.PubsubProviders[0].Azure.Namespace, qt.Equals, "my-servicebus-namespace") + + // Topic mapped from Azure Service Bus topic + topic, ok := parsedRuntime.PubsubTopics["encore-topic"] + c.Assert(ok, qt.IsTrue) + c.Assert(topic.ProviderName, qt.Equals, "azure-topic-name") + c.Assert(topic.ProviderID, qt.Equals, 0) + + // Subscription mapped from Azure Service Bus subscription + sub, ok := topic.Subscriptions["encore-subscription"] + c.Assert(ok, qt.IsTrue) + c.Assert(sub.ProviderName, qt.Equals, "azure-subscription-name") + c.Assert(sub.PushOnly, qt.IsFalse) + c.Assert(sub.GCP, qt.IsNil) +} diff --git a/runtimes/go/appruntime/infrasdk/metadata/azure_collector.go b/runtimes/go/appruntime/infrasdk/metadata/azure_collector.go new file mode 100644 index 0000000000..8bdd68b32d --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/metadata/azure_collector.go @@ -0,0 +1,79 @@ +//go:build !encore_no_azure + +package metadata + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + encore "encore.dev" +) + +// azureIMDSEndpoint is the Azure Instance Metadata Service endpoint. +// https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service +// Declared as a variable so that tests can override it to point at an httptest server. +var azureIMDSEndpoint = "http://169.254.169.254/metadata/instance?api-version=2021-02-01" + +func init() { + registerCollector(collectorDesc{ + name: "azure", + matches: func(envCloud string) bool { + return envCloud == encore.CloudAzure + }, + collect: func() (*ContainerMetadata, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, azureIMDSEndpoint, nil) + if err != nil { + return nil, fmt.Errorf("azure imds: create request: %w", err) + } + // The Metadata header is required by the Azure IMDS service. + req.Header.Set("Metadata", "true") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + // IMDS may be unavailable outside Azure; return empty metadata gracefully. + return &ContainerMetadata{}, nil + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("azure imds: read response: %w", err) + } + + var imds struct { + Compute struct { + Location string `json:"location"` + Name string `json:"name"` + ResourceGroupName string `json:"resourceGroupName"` + SubscriptionID string `json:"subscriptionId"` + VMID string `json:"vmId"` + } `json:"compute"` + } + if err := json.Unmarshal(body, &imds); err != nil { + return nil, fmt.Errorf("azure imds: unmarshal response: %w", err) + } + + // Map IMDS fields to ContainerMetadata: + // ServiceID → resource group (closest equivalent to an ECS service boundary) + // RevisionID → empty (no direct equivalent on Azure) + // InstanceID → last 8 chars of the VM/container unique ID + instanceID := imds.Compute.VMID + if len(instanceID) > 8 { + instanceID = instanceID[len(instanceID)-8:] + } + + return &ContainerMetadata{ + ServiceID: imds.Compute.ResourceGroupName, + RevisionID: "", + InstanceID: instanceID, + }, nil + }, + }) +} diff --git a/runtimes/go/appruntime/infrasdk/metadata/azure_collector_test.go b/runtimes/go/appruntime/infrasdk/metadata/azure_collector_test.go new file mode 100644 index 0000000000..b3b619eb04 --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/metadata/azure_collector_test.go @@ -0,0 +1,175 @@ +//go:build !encore_no_azure + +package metadata + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +// azureCollect looks up the azure collector from the registry and calls it. +// It also temporarily overrides azureIMDSEndpoint to use the provided URL. +func withIMDSEndpoint(t *testing.T, url string, fn func()) { + t.Helper() + orig := azureIMDSEndpoint + azureIMDSEndpoint = url + t.Cleanup(func() { azureIMDSEndpoint = orig }) + fn() +} + +// collectAzure finds the "azure" collector in the registry and runs it. +func collectAzure(t *testing.T) (*ContainerMetadata, error) { + t.Helper() + for _, c := range collectorRegistry { + if c.name == "azure" { + return c.collect() + } + } + t.Fatal("azure collector not found in registry") + return nil, nil +} + +// ---- successful IMDS response -------------------------------------------------------- + +func TestAzureCollector_SuccessfulResponse(t *testing.T) { + const ( + vmID = "aabbccdd-1234-5678-abcd-123456789012" + rgName = "my-resource-group" + wantInstID = "56789012" // last 8 chars of vmID + ) + + body := buildIMDSResponse(vmID, rgName) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Metadata") != "true" { + http.Error(w, "missing Metadata header", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + defer srv.Close() + + var got *ContainerMetadata + var err error + withIMDSEndpoint(t, srv.URL, func() { + got, err = collectAzure(t) + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.ServiceID != rgName { + t.Errorf("ServiceID: got %q, want %q", got.ServiceID, rgName) + } + if got.RevisionID != "" { + t.Errorf("RevisionID: got %q, want empty", got.RevisionID) + } + if got.InstanceID != wantInstID { + t.Errorf("InstanceID: got %q, want %q", got.InstanceID, wantInstID) + } +} + +// ---- field mapping tests ------------------------------------------------------------- + +func TestAzureCollector_FieldMapping(t *testing.T) { + tests := []struct { + name string + vmID string + rgName string + wantInstID string + }{ + { + name: "long vmId – last 8 chars used", + vmID: "00000000-0000-0000-0000-000099887766", + rgName: "prod-rg", + wantInstID: "99887766", + }, + { + name: "short vmId – used as-is", + vmID: "short", + rgName: "dev-rg", + wantInstID: "short", + }, + { + name: "exactly 8 chars vmId", + vmID: "12345678", + rgName: "qa-rg", + wantInstID: "12345678", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := buildIMDSResponse(tt.vmID, tt.rgName) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + defer srv.Close() + + var got *ContainerMetadata + var err error + withIMDSEndpoint(t, srv.URL, func() { + got, err = collectAzure(t) + }) + + if err != nil { + t.Fatalf("%s: unexpected error: %v", tt.name, err) + } + if got.ServiceID != tt.rgName { + t.Errorf("%s: ServiceID: got %q, want %q", tt.name, got.ServiceID, tt.rgName) + } + if got.InstanceID != tt.wantInstID { + t.Errorf("%s: InstanceID: got %q, want %q", tt.name, got.InstanceID, tt.wantInstID) + } + }) + } +} + +// ---- unreachable IMDS ---------------------------------------------------------------- + +func TestAzureCollector_IMDSUnreachable(t *testing.T) { + // Start and immediately close a server so the URL is valid but nothing listens. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) + url := srv.URL + srv.Close() + + var got *ContainerMetadata + var err error + withIMDSEndpoint(t, url, func() { + got, err = collectAzure(t) + }) + + if err != nil { + t.Fatalf("expected graceful empty return, got error: %v", err) + } + if got == nil { + t.Fatal("expected non-nil ContainerMetadata, got nil") + } + if got.ServiceID != "" || got.InstanceID != "" { + t.Errorf("expected empty metadata on IMDS failure, got %+v", got) + } +} + +// ---- helper -------------------------------------------------------------------------- + +// buildIMDSResponse constructs a minimal IMDS JSON response body. +func buildIMDSResponse(vmID, resourceGroupName string) []byte { + payload := map[string]interface{}{ + "compute": map[string]interface{}{ + "location": "eastus", + "name": "my-vm", + "resourceGroupName": resourceGroupName, + "subscriptionId": "sub-12345", + "vmId": vmID, + }, + } + b, err := json.Marshal(payload) + if err != nil { + panic(fmt.Sprintf("buildIMDSResponse: %v", err)) + } + return b +} diff --git a/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor.go b/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor.go new file mode 100644 index 0000000000..eff9c119c8 --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor.go @@ -0,0 +1,332 @@ +//go:build !encore_no_azure + +package azure + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/rs/zerolog" + + "encore.dev/appruntime/exported/config" + "encore.dev/appruntime/infrasdk/metadata" + "encore.dev/appruntime/infrasdk/metrics/system" + "encore.dev/appruntime/shared/nativehist" + "encore.dev/appruntime/shared/shutdown" + "encore.dev/metrics" +) + +// New creates a new Azure Monitor metrics exporter. +func New(svcs []string, cfg *config.AzureMonitorMetricsProvider, meta *metadata.ContainerMetadata, rootLogger zerolog.Logger) *Exporter { + return &Exporter{ + svcs: svcs, + cfg: cfg, + rootLogger: rootLogger, + containerMetaDims: metadata.MapMetadataLabels(meta, func(key, value string) dimKV { + return dimKV{key: key, value: value} + }), + } +} + +type dimKV struct { + key, value string +} + +// metricBatch groups series that share the same dimension names for a single +// Azure Monitor custom-metrics POST request. +type metricBatch struct { + dimNames []string + series []azureCustomMetricSeries +} + +// Exporter sends Encore metrics to Azure Monitor using the Custom Metrics REST API. +// https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-custom-overview +type Exporter struct { + svcs []string + cfg *config.AzureMonitorMetricsProvider + containerMetaDims []dimKV + rootLogger zerolog.Logger + + credMu sync.Mutex + cred *azidentity.DefaultAzureCredential +} + +func (x *Exporter) Shutdown(p *shutdown.Process) error { + return nil +} + +func (x *Exporter) Export(ctx context.Context, collected []metrics.CollectedMetric) error { + now := time.Now().UTC() + + batches := x.getMetricBatches(now, collected) + for name, b := range x.getSysBatches(now) { + batches[name] = b + } + + token, err := x.getToken(ctx) + if err != nil { + return fmt.Errorf("azure monitor: get auth token: %w", err) + } + + for metricName, batch := range batches { + if err := x.sendBatch(ctx, token, now, metricName, batch); err != nil { + return err + } + } + return nil +} + +// getMetricBatches converts collected Encore metrics into per-name batches ready for posting. +func (x *Exporter) getMetricBatches(now time.Time, collected []metrics.CollectedMetric) map[string]metricBatch { + result := make(map[string]metricBatch) + + for _, m := range collected { + // Build base dimension list: container metadata dims + metric label dims. + baseDims := make([]dimKV, 0, len(x.containerMetaDims)+len(m.Labels)) + baseDims = append(baseDims, x.containerMetaDims...) + for _, l := range m.Labels { + baseDims = append(baseDims, dimKV{key: l.Key, value: l.Value}) + } + + svcNum := m.Info.SvcNum() + + doAdd := func(s azureCustomMetricSeries, svcIdx uint16) { + dims := append(baseDims, dimKV{key: "service", value: x.svcs[svcIdx]}) + + dimNames := make([]string, len(dims)) + dimValues := make([]string, len(dims)) + for i, kv := range dims { + dimNames[i] = kv.key + dimValues[i] = kv.value + } + s.DimValues = dimValues + + b := result[m.Info.Name()] + if b.dimNames == nil { + b.dimNames = dimNames + } + b.series = append(b.series, s) + result[m.Info.Name()] = b + } + + scalarSeries := func(val float64) azureCustomMetricSeries { + return azureCustomMetricSeries{Sum: val, Count: 1, Min: val, Max: val} + } + + switch vals := m.Val.(type) { + case []float64: + if svcNum > 0 { + if m.Valid[0].Load() { + doAdd(scalarSeries(vals[0]), svcNum-1) + } + } else { + for i, val := range vals { + if m.Valid[i].Load() { + doAdd(scalarSeries(val), uint16(i)) + } + } + } + case []int64: + if svcNum > 0 { + if m.Valid[0].Load() { + doAdd(scalarSeries(float64(vals[0])), svcNum-1) + } + } else { + for i, val := range vals { + if m.Valid[i].Load() { + doAdd(scalarSeries(float64(val)), uint16(i)) + } + } + } + case []uint64: + if svcNum > 0 { + if m.Valid[0].Load() { + doAdd(scalarSeries(float64(vals[0])), svcNum-1) + } + } else { + for i, val := range vals { + if m.Valid[i].Load() { + doAdd(scalarSeries(float64(val)), uint16(i)) + } + } + } + case []time.Duration: + if svcNum > 0 { + if m.Valid[0].Load() { + doAdd(scalarSeries(float64(vals[0]/time.Second)), svcNum-1) + } + } else { + for i, val := range vals { + if m.Valid[i].Load() { + doAdd(scalarSeries(float64(val/time.Second)), uint16(i)) + } + } + } + case []*nativehist.Histogram: + if svcNum > 0 { + if m.Valid[0].Load() && vals[0] != nil { + st := vals[0].Stats() + doAdd(azureCustomMetricSeries{ + Sum: st.Sum, + Count: int(st.Count), + Min: st.Min, + Max: st.Max, + }, svcNum-1) + } + } else { + for i, h := range vals { + if m.Valid[i].Load() && h != nil { + st := h.Stats() + doAdd(azureCustomMetricSeries{ + Sum: st.Sum, + Count: int(st.Count), + Min: st.Min, + Max: st.Max, + }, uint16(i)) + } + } + } + default: + x.rootLogger.Error().Msgf("encore: internal error: unknown value type %T for metric %s", m.Val, m.Info.Name()) + } + } + return result +} + +// getSysBatches returns batches for Go runtime system metrics. +func (x *Exporter) getSysBatches(now time.Time) map[string]metricBatch { + sysMetrics := system.ReadSysMetrics(x.rootLogger) + + dimNames := make([]string, len(x.containerMetaDims)) + dimValues := make([]string, len(x.containerMetaDims)) + for i, kv := range x.containerMetaDims { + dimNames[i] = kv.key + dimValues[i] = kv.value + } + + makeBatch := func(val uint64) metricBatch { + f := float64(val) + return metricBatch{ + dimNames: dimNames, + series: []azureCustomMetricSeries{ + {DimValues: dimValues, Sum: f, Count: 1, Min: f, Max: f}, + }, + } + } + + return map[string]metricBatch{ + system.MetricNameHeapObjectsBytes: makeBatch(sysMetrics[system.MetricNameHeapObjectsBytes]), + system.MetricNameGoroutines: makeBatch(sysMetrics[system.MetricNameGoroutines]), + } +} + +// azureCustomMetricPayload is the JSON body for the Azure Monitor custom metrics REST API. +type azureCustomMetricPayload struct { + Time string `json:"time"` + Data azureCustomMetricData `json:"data"` +} + +type azureCustomMetricData struct { + BaseData azureCustomMetricBaseData `json:"baseData"` +} + +type azureCustomMetricBaseData struct { + Metric string `json:"metric"` + Namespace string `json:"namespace"` + DimNames []string `json:"dimNames,omitempty"` + Series []azureCustomMetricSeries `json:"series"` +} + +type azureCustomMetricSeries struct { + DimValues []string `json:"dimValues,omitempty"` + Sum float64 `json:"sum"` + Count int `json:"count"` + Min float64 `json:"min"` + Max float64 `json:"max"` +} + +func (x *Exporter) sendBatch(ctx context.Context, token string, now time.Time, metricName string, batch metricBatch) error { + if len(batch.series) == 0 { + return nil + } + + payload := azureCustomMetricPayload{ + Time: now.Format(time.RFC3339), + Data: azureCustomMetricData{ + BaseData: azureCustomMetricBaseData{ + Metric: metricName, + Namespace: x.cfg.Namespace, + DimNames: batch.dimNames, + Series: batch.series, + }, + }, + } + + body, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("azure monitor: marshal payload for metric %s: %w", metricName, err) + } + + url := fmt.Sprintf( + "https://%s.monitoring.azure.com/subscriptions/%s/resourceGroups/%s/providers/%s/%s/metrics", + x.cfg.Location, + x.cfg.SubscriptionID, + x.cfg.ResourceGroup, + x.cfg.ResourceNamespace, + x.cfg.ResourceName, + ) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("azure monitor: create request for metric %s: %w", metricName, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("azure monitor: send metric %s: %w", metricName, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("azure monitor: unexpected status %d for metric %s", resp.StatusCode, metricName) + } + return nil +} + +// getToken returns a fresh bearer token for the Azure Monitor scope. +// The azidentity credential caches the token and refreshes it automatically. +func (x *Exporter) getToken(ctx context.Context) (string, error) { + cred, err := x.getCred() + if err != nil { + return "", err + } + tok, err := cred.GetToken(ctx, policy.TokenRequestOptions{ + Scopes: []string{"https://monitoring.azure.com/.default"}, + }) + if err != nil { + return "", err + } + return tok.Token, nil +} + +func (x *Exporter) getCred() (*azidentity.DefaultAzureCredential, error) { + x.credMu.Lock() + defer x.credMu.Unlock() + if x.cred == nil { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("create Azure credential: %w", err) + } + x.cred = cred + } + return x.cred, nil +} diff --git a/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor_test.go b/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor_test.go new file mode 100644 index 0000000000..6229335a86 --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/metrics/azure/azure_monitor_test.go @@ -0,0 +1,304 @@ +//go:build !encore_no_azure + +package azure + +import ( + "encoding/json" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/rs/zerolog" + + "encore.dev/appruntime/infrasdk/metadata" + "encore.dev/appruntime/infrasdk/metrics/system" + "encore.dev/metrics" +) + +// metricInfo is a test implementation of metrics.MetricInfo. +type metricInfo struct { + name string + typ metrics.MetricType + svcNum uint16 +} + +func (m metricInfo) Name() string { return m.name } +func (m metricInfo) Type() metrics.MetricType { return m.typ } +func (m metricInfo) SvcNum() uint16 { return m.svcNum } + +// validBools returns a slice of atomic.Bool, all set to true. +func validBools(n int) []atomic.Bool { + v := make([]atomic.Bool, n) + for i := range v { + v[i].Store(true) + } + return v +} + +// newTestExporter creates an Exporter with no real Azure config, suitable for +// testing the pure batch-building logic. +func newTestExporter(svcs []string, meta *metadata.ContainerMetadata) *Exporter { + return New(svcs, nil, meta, zerolog.New(io.Discard)) +} + +// ---- getMetricBatches tests ----------------------------------------------------------- + +func TestGetMetricBatches_Counter(t *testing.T) { + now := time.Now() + svcs := []string{"svc-a", "svc-b"} + meta := &metadata.ContainerMetadata{ + ServiceID: "rg-prod", + InstanceID: "inst-1", + } + + x := newTestExporter(svcs, meta) + collected := []metrics.CollectedMetric{ + { + Info: metricInfo{"http_requests_total", metrics.CounterType, 1}, + Val: []int64{42}, + Valid: validBools(1), + }, + } + + batches := x.getMetricBatches(now, collected) + batch, ok := batches["http_requests_total"] + if !ok { + t.Fatal("expected batch for http_requests_total, got none") + } + if len(batch.series) != 1 { + t.Fatalf("expected 1 series, got %d", len(batch.series)) + } + got := batch.series[0] + if got.Sum != 42 { + t.Errorf("Sum: got %v, want 42", got.Sum) + } + // The last dim is always "service". + last := got.DimValues[len(got.DimValues)-1] + if last != "svc-a" { + t.Errorf("service dim: got %q, want %q", last, "svc-a") + } +} + +func TestGetMetricBatches_MultipleServices(t *testing.T) { + now := time.Now() + svcs := []string{"svc-a", "svc-b"} + meta := &metadata.ContainerMetadata{} + + x := newTestExporter(svcs, meta) + collected := []metrics.CollectedMetric{ + { + // svcNum=0 means iterate all services. + Info: metricInfo{"active_conns", metrics.GaugeType, 0}, + Val: []float64{10, 20}, + Valid: validBools(2), + }, + } + + batches := x.getMetricBatches(now, collected) + batch, ok := batches["active_conns"] + if !ok { + t.Fatal("expected batch for active_conns") + } + if len(batch.series) != 2 { + t.Fatalf("expected 2 series (one per service), got %d", len(batch.series)) + } + + // Verify each service has its data. + svcValues := map[string]float64{} + for _, s := range batch.series { + svc := s.DimValues[len(s.DimValues)-1] + svcValues[svc] = s.Sum + } + if svcValues["svc-a"] != 10 { + t.Errorf("svc-a: got %v, want 10", svcValues["svc-a"]) + } + if svcValues["svc-b"] != 20 { + t.Errorf("svc-b: got %v, want 20", svcValues["svc-b"]) + } +} + +func TestGetMetricBatches_Labels(t *testing.T) { + now := time.Now() + svcs := []string{"svc-a"} + meta := &metadata.ContainerMetadata{} + + x := newTestExporter(svcs, meta) + collected := []metrics.CollectedMetric{ + { + Info: metricInfo{"cache_hits", metrics.CounterType, 1}, + Labels: []metrics.KeyValue{{Key: "cache_type", Value: "redis"}}, + Val: []float64{5}, + Valid: validBools(1), + }, + } + + batches := x.getMetricBatches(now, collected) + batch, ok := batches["cache_hits"] + if !ok { + t.Fatal("expected batch for cache_hits") + } + if len(batch.series) != 1 { + t.Fatalf("expected 1 series, got %d", len(batch.series)) + } + + // Dim names must contain the label key and "service". + foundLabel, foundService := false, false + for _, name := range batch.dimNames { + if name == "cache_type" { + foundLabel = true + } + if name == "service" { + foundService = true + } + } + if !foundLabel { + t.Errorf("dim names %v missing cache_type", batch.dimNames) + } + if !foundService { + t.Errorf("dim names %v missing service", batch.dimNames) + } +} + +func TestGetMetricBatches_Empty(t *testing.T) { + now := time.Now() + x := newTestExporter([]string{"svc"}, &metadata.ContainerMetadata{}) + + batches := x.getMetricBatches(now, nil) + if len(batches) != 0 { + t.Errorf("expected empty batches for nil input, got %d entries", len(batches)) + } + + batches2 := x.getMetricBatches(now, []metrics.CollectedMetric{}) + if len(batches2) != 0 { + t.Errorf("expected empty batches for empty slice, got %d entries", len(batches2)) + } +} + +func TestGetMetricBatches_InvalidMetricSkipped(t *testing.T) { + now := time.Now() + svcs := []string{"svc-a"} + x := newTestExporter(svcs, &metadata.ContainerMetadata{}) + + // Valid[0] is false → the metric should be skipped. + invalid := make([]atomic.Bool, 1) + invalid[0].Store(false) + + collected := []metrics.CollectedMetric{ + { + Info: metricInfo{"skipped_metric", metrics.CounterType, 1}, + Val: []int64{99}, + Valid: invalid, + }, + } + + batches := x.getMetricBatches(now, collected) + if len(batches) != 0 { + t.Errorf("expected no batches for invalid metric, got %d", len(batches)) + } +} + +// ---- getSysBatches tests -------------------------------------------------------------- + +func TestGetSysBatches(t *testing.T) { + x := newTestExporter([]string{"svc"}, &metadata.ContainerMetadata{ + ServiceID: "rg", + InstanceID: "i1", + }) + + batches := x.getSysBatches(time.Now()) + + if _, ok := batches[system.MetricNameHeapObjectsBytes]; !ok { + t.Errorf("getSysBatches missing %s", system.MetricNameHeapObjectsBytes) + } + if _, ok := batches[system.MetricNameGoroutines]; !ok { + t.Errorf("getSysBatches missing %s", system.MetricNameGoroutines) + } + + for name, batch := range batches { + if len(batch.series) != 1 { + t.Errorf("%s: expected 1 series, got %d", name, len(batch.series)) + } + } +} + +// ---- payload serialization tests ----------------------------------------------------- + +func TestPayloadSerialization(t *testing.T) { + now := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC) + namespace := "Encore/Metrics" + metricName := "http_requests_total" + + payload := azureCustomMetricPayload{ + Time: now.Format(time.RFC3339), + Data: azureCustomMetricData{ + BaseData: azureCustomMetricBaseData{ + Metric: metricName, + Namespace: namespace, + DimNames: []string{"service", "region"}, + Series: []azureCustomMetricSeries{ + { + DimValues: []string{"svc-a", "eastus"}, + Sum: 42, + Count: 1, + Min: 42, + Max: 42, + }, + }, + }, + }, + } + + data, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + var got map[string]interface{} + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("unmarshal result: %v", err) + } + + // Verify top-level shape. + if _, ok := got["time"]; !ok { + t.Error("payload missing 'time' field") + } + dataObj, ok := got["data"].(map[string]interface{}) + if !ok { + t.Fatalf("payload 'data' field is not an object; got %T", got["data"]) + } + baseData, ok := dataObj["baseData"].(map[string]interface{}) + if !ok { + t.Fatalf("data.baseData is not an object; got %T", dataObj["baseData"]) + } + + if baseData["metric"] != metricName { + t.Errorf("metric: got %v, want %v", baseData["metric"], metricName) + } + if baseData["namespace"] != namespace { + t.Errorf("namespace: got %v, want %v", baseData["namespace"], namespace) + } + + series, ok := baseData["series"].([]interface{}) + if !ok || len(series) == 0 { + t.Fatalf("series is missing or empty: %v", baseData["series"]) + } + s, ok := series[0].(map[string]interface{}) + if !ok { + t.Fatalf("first series item is not an object; got %T", series[0]) + } + + for _, required := range []string{"sum", "count", "min", "max"} { + if _, ok := s[required]; !ok { + t.Errorf("series item missing %q field; got keys: %v", required, keys(s)) + } + } +} + +func keys(m map[string]interface{}) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + return ks +} diff --git a/runtimes/go/appruntime/infrasdk/metrics/azure_monitor_exporter.go b/runtimes/go/appruntime/infrasdk/metrics/azure_monitor_exporter.go new file mode 100644 index 0000000000..95f1938883 --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/metrics/azure_monitor_exporter.go @@ -0,0 +1,27 @@ +//go:build !encore_no_azure + +package metrics + +import ( + "encore.dev/appruntime/exported/config" + "encore.dev/appruntime/infrasdk/metadata" + "encore.dev/appruntime/infrasdk/metrics/azure" +) + +func init() { + registerProvider(providerDesc{ + name: "azure_monitor", + matches: func(cfg *config.Metrics) bool { + return cfg.AzureMonitor != nil + }, + newExporter: func(m *Manager) exporter { + containerMetadata, err := metadata.GetContainerMetadata(m.runtime) + if err != nil { + m.rootLogger.Err(err).Msg("unable to initialize metrics exporter: error getting container metadata") + return nil + } + + return azure.New(m.static.BundledServices, m.runtime.Metrics.AzureMonitor, containerMetadata, m.rootLogger) + }, + }) +} diff --git a/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault.go b/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault.go new file mode 100644 index 0000000000..f54b075deb --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault.go @@ -0,0 +1,46 @@ +//go:build !encore_no_azure + +package secrets + +import ( + "context" + "fmt" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" + + "encore.dev/appruntime/exported/config/infra" +) + +func init() { + newAzureKVProvider = func(cfg *infra.AzureKeyVaultSecretsProvider) (remoteSecretsProvider, error) { + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("azure key vault: create credential: %w", err) + } + client, err := azsecrets.NewClient(cfg.VaultURL, cred, nil) + if err != nil { + return nil, fmt.Errorf("azure key vault: create client: %w", err) + } + return &azureKVProvider{client: client}, nil + } +} + +// azureKVProvider fetches secrets from Azure Key Vault. +// Encore secret names map directly to Key Vault secret names. +type azureKVProvider struct { + client *azsecrets.Client +} + +// FetchSecret retrieves the latest version of a secret from Azure Key Vault. +func (p *azureKVProvider) FetchSecret(ctx context.Context, name string) (string, error) { + // Pass an empty version string to retrieve the latest enabled version. + resp, err := p.client.GetSecret(ctx, name, "", nil) + if err != nil { + return "", fmt.Errorf("azure key vault: get secret %q: %w", name, err) + } + if resp.Value == nil { + return "", fmt.Errorf("azure key vault: secret %q returned no value", name) + } + return *resp.Value, nil +} diff --git a/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault_test.go b/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault_test.go new file mode 100644 index 0000000000..28e4c8b964 --- /dev/null +++ b/runtimes/go/appruntime/infrasdk/secrets/azure_keyvault_test.go @@ -0,0 +1,238 @@ +//go:build !encore_no_azure + +package secrets + +import ( + "context" + "crypto/tls" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets" + qt "github.com/frankban/quicktest" +) + +// TestFetchSecret_Success tests that a valid secret value is returned from Key Vault. +func TestFetchSecret_Success(t *testing.T) { + c := qt.New(t) + + const secretName = "test-secret" + const secretValue = "super-secret-value" + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Azure SDK sends path like /secrets/test-secret/ with trailing slash + if r.URL.Path != "/secrets/"+secretName+"/" && r.URL.Path != "/secrets/"+secretName { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "value": secretValue, + "id": "https://test.vault.azure.net/secrets/" + secretName, + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + got, err := provider.FetchSecret(context.Background(), secretName) + c.Assert(err, qt.IsNil) + c.Assert(got, qt.Equals, secretValue) +} + +// TestFetchSecret_NotFound tests that a 404 from Key Vault returns an error. +func TestFetchSecret_NotFound(t *testing.T) { + c := qt.New(t) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "error": map[string]interface{}{ + "code": "SecretNotFound", + "message": "Secret not found", + }, + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + _, err := provider.FetchSecret(context.Background(), "nonexistent") + c.Assert(err, qt.Not(qt.IsNil)) +} + +// TestFetchSecret_NilValue tests that a response with nil Value returns an error. +func TestFetchSecret_NilValue(t *testing.T) { + c := qt.New(t) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "id": "https://test.vault.azure.net/secrets/test", + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + _, err := provider.FetchSecret(context.Background(), "test") + c.Assert(err, qt.Not(qt.IsNil)) + c.Assert(err.Error(), qt.Contains, "returned no value") +} + +// TestFetchSecret_EmptyValue tests that a response with empty string value is handled. +func TestFetchSecret_EmptyValue(t *testing.T) { + c := qt.New(t) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "value": "", + "id": "https://test.vault.azure.net/secrets/test", + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + got, err := provider.FetchSecret(context.Background(), "test") + c.Assert(err, qt.IsNil) + c.Assert(got, qt.Equals, "") +} + +// TestFetchSecret_ContextCanceled tests that context cancellation is handled. +func TestFetchSecret_ContextCanceled(t *testing.T) { + c := qt.New(t) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := provider.FetchSecret(ctx, "test") + c.Assert(err, qt.Not(qt.IsNil)) +} + +// TestFetchSecret_SDKError tests that SDK errors are propagated. +func TestFetchSecret_SDKError(t *testing.T) { + c := qt.New(t) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "error": map[string]interface{}{ + "code": "InternalServerError", + "message": "Internal server error", + }, + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + _, err := provider.FetchSecret(context.Background(), "test") + c.Assert(err, qt.Not(qt.IsNil)) +} + +// TestFetchSecret_MultipleSecrets tests fetching multiple different secrets. +func TestFetchSecret_MultipleSecrets(t *testing.T) { + c := qt.New(t) + + secrets := map[string]string{ + "db-password": "password123", + "api-key": "key456", + "token": "token789", + } + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for name, value := range secrets { + // Azure SDK sends path with trailing slash + if r.URL.Path == "/secrets/"+name+"/" || r.URL.Path == "/secrets/"+name { + w.Header().Set("Content-Type", "application/json") + resp := map[string]interface{}{ + "value": value, + "id": "https://test.vault.azure.net/secrets/" + name, + } + _ = json.NewEncoder(w).Encode(resp) + return + } + } + http.Error(w, "not found", http.StatusNotFound) + })) + defer srv.Close() + + provider := createTestProvider(t, srv) + + for name, expected := range secrets { + got, err := provider.FetchSecret(context.Background(), name) + c.Assert(err, qt.IsNil, qt.Commentf("secret %q", name)) + c.Assert(got, qt.Equals, expected, qt.Commentf("secret %q", name)) + } +} + +// createTestProvider creates an azureKVProvider configured to use the test server. +// It uses a fake credential that doesn't require real Azure authentication. +func createTestProvider(t *testing.T, srv *httptest.Server) *azureKVProvider { + t.Helper() + + cred := &fakeCredential{} + + // Configure the client to skip TLS verification for test servers + opts := &azsecrets.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + }, + }, + } + + client, err := azsecrets.NewClient(srv.URL, cred, opts) + if err != nil { + t.Fatalf("create test client: %v", err) + } + + return &azureKVProvider{client: client} +} + +// fakeCredential is a fake Azure credential for testing that returns a dummy token. +type fakeCredential struct{} + +func (f *fakeCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { + expiresOn := time.Now().Add(time.Hour) + return azcore.AccessToken{ + Token: "fake-token-for-testing", + ExpiresOn: expiresOn, + }, nil +} + +// TestNewAzureKVProvider tests the provider initialization through the init function. +func TestNewAzureKVProvider(t *testing.T) { + c := qt.New(t) + + // Test that newAzureKVProvider was set by the init function + c.Assert(newAzureKVProvider, qt.Not(qt.IsNil)) + + // We can't easily test the real newAzureKVProvider function here because it + // attempts to create a DefaultAzureCredential, which requires real Azure + // authentication or environment variables. This test just verifies that the + // init function registered the provider function. +} diff --git a/runtimes/go/appruntime/infrasdk/secrets/manager_internal.go b/runtimes/go/appruntime/infrasdk/secrets/manager_internal.go index f0bad06c4a..a8d235071b 100644 --- a/runtimes/go/appruntime/infrasdk/secrets/manager_internal.go +++ b/runtimes/go/appruntime/infrasdk/secrets/manager_internal.go @@ -3,6 +3,7 @@ package secrets import ( "bytes" "compress/gzip" + "context" "encoding/base64" "fmt" "io" @@ -10,26 +11,54 @@ import ( "maps" "os" "strings" + "sync" + "time" "encore.dev/appruntime/exported/config" + "encore.dev/appruntime/exported/config/infra" "encore.dev/appruntime/shared/cfgutil" ) +// remoteSecretsProvider is implemented by cloud-specific secret backends (e.g. Azure Key Vault). +type remoteSecretsProvider interface { + FetchSecret(ctx context.Context, name string) (string, error) +} + +// newAzureKVProvider is set by azure_keyvault.go when built with Azure support. +// If nil, the Azure Key Vault provider is unavailable. +var newAzureKVProvider func(cfg *infra.AzureKeyVaultSecretsProvider) (remoteSecretsProvider, error) + type Manager struct { - cfg *config.Runtime - secrets map[string]string + cfg *config.Runtime + secrets map[string]string + remote remoteSecretsProvider + remoteCache sync.Map // map[string]string — caches values already fetched from remote } func NewManager(cfg *config.Runtime, infraCfgEnv, appSecretsEnv string) *Manager { secrets := parse(appSecretsEnv) + var remote remoteSecretsProvider if infraCfgEnv != "" { - cfg, err := config.LoadInfraConfig(infraCfgEnv) + infraCfg, err := config.LoadInfraConfig(infraCfgEnv) if err != nil { log.Fatalln("encore: could not read infra config", err) } - maps.Copy(secrets, cfg.Secrets.GetSecrets()) + maps.Copy(secrets, infraCfg.Secrets.GetSecrets()) + + // Wire up the remote secrets provider if one is configured. + if p := infraCfg.SecretsProvider; p != nil { + if kv := p.AzureKeyVault; kv != nil { + if newAzureKVProvider == nil { + log.Fatalln("encore: Azure Key Vault secrets provider is configured but Azure support was not compiled in (built with encore_no_azure?)") + } + remote, err = newAzureKVProvider(kv) + if err != nil { + log.Fatalln("encore: could not initialize Azure Key Vault secrets provider:", err) + } + } + } } - return &Manager{cfg: cfg, secrets: secrets} + return &Manager{cfg: cfg, secrets: secrets, remote: remote} } // Load loads a secret. @@ -38,6 +67,21 @@ func (mgr *Manager) Load(key string, inService string) string { return val } + // Try the remote provider (e.g. Azure Key Vault) with a local in-memory cache. + if mgr.remote != nil { + if cached, ok := mgr.remoteCache.Load(key); ok { + return cached.(string) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if val, err := mgr.remote.FetchSecret(ctx, key); err == nil { + mgr.remoteCache.Store(key, val) + return val + } else { + fmt.Fprintf(os.Stderr, "encore: error fetching secret %q from remote provider: %v\n", key, err) + } + } + // For anything but local development or a gateway, a missing secret is a fatal error. if mgr.cfg.EnvCloud != "local" && cfgutil.IsHostedService(mgr.cfg, inService) { fmt.Fprintln(os.Stderr, "encore: could not find secret", key) diff --git a/runtimes/go/appruntime/shared/cloudtrace/azure.go b/runtimes/go/appruntime/shared/cloudtrace/azure.go new file mode 100644 index 0000000000..b09e1715ae --- /dev/null +++ b/runtimes/go/appruntime/shared/cloudtrace/azure.go @@ -0,0 +1,87 @@ +package cloudtrace + +import ( + "os" + "strings" + "sync" +) + +var ( + azureInstrumentationKey string + azureConnectionString string + azureResourceLoad sync.Once +) + +// AzureInstrumentationKey returns the Azure Application Insights instrumentation key. +// Returns empty string if not configured. +func AzureInstrumentationKey() string { + azureResourceLoad.Do(loadAzureResourceInfo) + return azureInstrumentationKey +} + +// AzureConnectionString returns the Azure Application Insights connection string. +// Returns empty string if not configured. +func AzureConnectionString() string { + azureResourceLoad.Do(loadAzureResourceInfo) + return azureConnectionString +} + +func loadAzureResourceInfo() { + // recover from any panics + defer func() { + if r := recover(); r != nil { + azureConnectionString = "" + azureInstrumentationKey = "" + } + }() + + // Check connection string first (preferred over standalone instrumentation key) + connStr := azureConnectionStringFromEnv() + if connStr != "" { + azureConnectionString = connStr + azureInstrumentationKey = extractInstrumentationKeyFromConnStr(connStr) + return + } + + // Fall back to standalone instrumentation key + azureInstrumentationKey = azureInstrumentationKeyFromEnv() +} + +func azureConnectionStringFromEnv() string { + for _, key := range []string{ + "APPLICATIONINSIGHTS_CONNECTION_STRING", + "applicationinsights_connection_string", + } { + if v := os.Getenv(key); v != "" { + return v + } + } + return "" +} + +func azureInstrumentationKeyFromEnv() string { + for _, key := range []string{ + "APPINSIGHTS_INSTRUMENTATIONKEY", + "appinsights_instrumentationkey", + } { + if v := os.Getenv(key); v != "" { + return v + } + } + return "" +} + +// extractInstrumentationKeyFromConnStr parses the InstrumentationKey from an +// Application Insights connection string. +// Format: "InstrumentationKey=;IngestionEndpoint=https://...;..." +func extractInstrumentationKeyFromConnStr(connStr string) string { + for _, part := range strings.Split(connStr, ";") { + if strings.EqualFold(strings.TrimSpace(strings.SplitN(part, "=", 2)[0]), "InstrumentationKey") { + kv := strings.SplitN(part, "=", 2) + if len(kv) == 2 { + return strings.TrimSpace(kv[1]) + } + } + } + return "" +} diff --git a/runtimes/go/appruntime/shared/cloudtrace/azure_test.go b/runtimes/go/appruntime/shared/cloudtrace/azure_test.go new file mode 100644 index 0000000000..828c076b12 --- /dev/null +++ b/runtimes/go/appruntime/shared/cloudtrace/azure_test.go @@ -0,0 +1,318 @@ +package cloudtrace + +import ( + "net/http/httptest" + "testing" +) + +// TestAzureConnectionStringFromEnv tests the private azureConnectionStringFromEnv helper +func TestAzureConnectionStringFromEnv(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expected string + }{ + { + name: "empty environment", + envVars: map[string]string{}, + expected: "", + }, + { + name: "uppercase env var set", + envVars: map[string]string{ + "APPLICATIONINSIGHTS_CONNECTION_STRING": "InstrumentationKey=abc123;IngestionEndpoint=https://example.com", + }, + expected: "InstrumentationKey=abc123;IngestionEndpoint=https://example.com", + }, + { + name: "lowercase env var set", + envVars: map[string]string{ + "applicationinsights_connection_string": "InstrumentationKey=xyz789;IngestionEndpoint=https://example.com", + }, + expected: "InstrumentationKey=xyz789;IngestionEndpoint=https://example.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set env vars + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + result := azureConnectionStringFromEnv() + if result != tt.expected { + t.Errorf("azureConnectionStringFromEnv() = %q, want %q", result, tt.expected) + } + }) + } +} + +// TestAzureInstrumentationKeyFromEnv tests the private azureInstrumentationKeyFromEnv helper +func TestAzureInstrumentationKeyFromEnv(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expected string + }{ + { + name: "empty environment", + envVars: map[string]string{}, + expected: "", + }, + { + name: "uppercase env var set", + envVars: map[string]string{ + "APPINSIGHTS_INSTRUMENTATIONKEY": "abc123-def456-ghi789", + }, + expected: "abc123-def456-ghi789", + }, + { + name: "lowercase env var set", + envVars: map[string]string{ + "appinsights_instrumentationkey": "xyz789-uvw456-rst123", + }, + expected: "xyz789-uvw456-rst123", + }, + { + name: "uppercase takes precedence when both set", + envVars: map[string]string{ + "APPINSIGHTS_INSTRUMENTATIONKEY": "uppercase-key", + }, + expected: "uppercase-key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set env vars + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + result := azureInstrumentationKeyFromEnv() + if result != tt.expected { + t.Errorf("azureInstrumentationKeyFromEnv() = %q, want %q", result, tt.expected) + } + }) + } +} + +// TestExtractInstrumentationKeyFromConnStr tests the private extractInstrumentationKeyFromConnStr helper +func TestExtractInstrumentationKeyFromConnStr(t *testing.T) { + tests := []struct { + name string + connStr string + expected string + }{ + { + name: "empty string", + connStr: "", + expected: "", + }, + { + name: "missing key", + connStr: "IngestionEndpoint=https://example.com;LiveEndpoint=https://example.com", + expected: "", + }, + { + name: "full connection string with key first", + connStr: "InstrumentationKey=abc123;IngestionEndpoint=https://example.com", + expected: "abc123", + }, + { + name: "key appears later in string", + connStr: "IngestionEndpoint=https://example.com;InstrumentationKey=xyz789;LiveEndpoint=https://example.com", + expected: "xyz789", + }, + { + name: "connection string with extra spaces", + connStr: "IngestionEndpoint=https://example.com; InstrumentationKey = abc123 ;LiveEndpoint=https://example.com", + expected: "abc123", + }, + { + name: "mixed case key name lowercase", + connStr: "instrumentationkey=abc123;IngestionEndpoint=https://example.com", + expected: "abc123", + }, + { + name: "mixed case key name uppercase", + connStr: "INSTRUMENTATIONKEY=abc123;IngestionEndpoint=https://example.com", + expected: "abc123", + }, + { + name: "mixed case key name camelCase", + connStr: "instrumentationKey=abc123;IngestionEndpoint=https://example.com", + expected: "abc123", + }, + { + name: "key with no value", + connStr: "InstrumentationKey=;IngestionEndpoint=https://example.com", + expected: "", + }, + { + name: "key without equals", + connStr: "InstrumentationKey;IngestionEndpoint=https://example.com", + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := extractInstrumentationKeyFromConnStr(tt.connStr) + if result != tt.expected { + t.Errorf("extractInstrumentationKeyFromConnStr(%q) = %q, want %q", tt.connStr, result, tt.expected) + } + }) + } +} + +// TestStructuredLogFields_AzureTraceparent tests the Azure log field enrichment in StructuredLogFields +// We need to test this carefully because AzureInstrumentationKey() uses sync.Once. +// We'll directly set the package-level variable to simulate the instrumentation key being configured. +func TestStructuredLogFields_AzureTraceparent(t *testing.T) { + // Save original values + origKey := azureInstrumentationKey + origConnStr := azureConnectionString + defer func() { + azureInstrumentationKey = origKey + azureConnectionString = origConnStr + }() + + tests := []struct { + name string + traceparent string + instrumentationKey string + expectOperationID bool + expectOperationParentID bool + expectedTraceID string + expectedParentIDPattern string + }{ + { + name: "no traceparent header", + traceparent: "", + instrumentationKey: "test-key", + expectOperationID: false, + expectOperationParentID: false, + }, + { + name: "valid traceparent but no instrumentation key", + traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + instrumentationKey: "", + expectOperationID: false, + expectOperationParentID: false, + }, + { + name: "valid traceparent with instrumentation key", + traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + instrumentationKey: "test-key-123", + expectOperationID: true, + expectOperationParentID: false, // parseTraceParent doesn't extract span ID + expectedTraceID: "4bf92f3577b34da6a3ce929d0e0e4736", + }, + { + name: "valid traceparent with zero span ID", + traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01", + instrumentationKey: "test-key-456", + expectOperationID: true, + expectOperationParentID: false, + expectedTraceID: "4bf92f3577b34da6a3ce929d0e0e4736", + }, + { + name: "another valid traceparent", + traceparent: "00-12345678901234567890123456789012-abcdef1234567890-00", + instrumentationKey: "another-key", + expectOperationID: true, + expectOperationParentID: false, // parseTraceParent doesn't extract span ID + expectedTraceID: "12345678901234567890123456789012", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Directly set the package-level instrumentation key variable + azureInstrumentationKey = tt.instrumentationKey + + // Create a fresh request with the traceparent header + req := httptest.NewRequest("GET", "http://example.com", nil) + if tt.traceparent != "" { + req.Header.Set("traceparent", tt.traceparent) + } + + // Call StructuredLogFields + fields := StructuredLogFields(req) + + // Check operation_Id + if tt.expectOperationID { + if operationID, ok := fields["operation_Id"]; !ok { + t.Errorf("expected operation_Id field to be set") + } else if operationID != tt.expectedTraceID { + t.Errorf("operation_Id = %q, want %q", operationID, tt.expectedTraceID) + } else if len(operationID) != 32 { + t.Errorf("operation_Id length = %d, want 32 (16-byte trace ID as hex)", len(operationID)) + } + } else { + if _, ok := fields["operation_Id"]; ok { + t.Errorf("expected operation_Id field to NOT be set") + } + } + + // Check operation_ParentId + if tt.expectOperationParentID { + if operationParentID, ok := fields["operation_ParentId"]; !ok { + t.Errorf("expected operation_ParentId field to be set") + } else if operationParentID != tt.expectedParentIDPattern { + t.Errorf("operation_ParentId = %q, want %q", operationParentID, tt.expectedParentIDPattern) + } + } else { + if _, ok := fields["operation_ParentId"]; ok { + t.Errorf("expected operation_ParentId field to NOT be set") + } + } + + // Ensure GCP fields are NOT set when no X-Cloud-Trace-Context header + if _, ok := fields["logging.googleapis.com/trace"]; ok { + t.Errorf("expected no GCP trace field when X-Cloud-Trace-Context header is not present") + } + if _, ok := fields["logging.googleapis.com/spanId"]; ok { + t.Errorf("expected no GCP spanId field when X-Cloud-Trace-Context header is not present") + } + }) + } +} + +// TestStructuredLogFields_NilRequest ensures StructuredLogFields handles nil request gracefully +func TestStructuredLogFields_NilRequest(t *testing.T) { + fields := StructuredLogFields(nil) + if fields != nil { + t.Errorf("StructuredLogFields(nil) = %v, want nil", fields) + } +} + +// TestStructuredLogFields_AzureAndGCPIsolation ensures Azure and GCP fields don't interfere +func TestStructuredLogFields_AzureAndGCPIsolation(t *testing.T) { + // Save original values + origKey := azureInstrumentationKey + defer func() { + azureInstrumentationKey = origKey + }() + + // Set instrumentation key for Azure + azureInstrumentationKey = "azure-key" + + // Create request with only Azure traceparent header (no GCP header) + req := httptest.NewRequest("GET", "http://example.com", nil) + req.Header.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01") + + fields := StructuredLogFields(req) + + // Should have Azure fields + if _, ok := fields["operation_Id"]; !ok { + t.Errorf("expected operation_Id field when traceparent header is set") + } + + // Should NOT have GCP fields + if _, ok := fields["logging.googleapis.com/trace"]; ok { + t.Errorf("expected no GCP trace field when only traceparent header is set") + } +} diff --git a/runtimes/go/appruntime/shared/cloudtrace/logfields.go b/runtimes/go/appruntime/shared/cloudtrace/logfields.go index 0d7f70a0c8..4ca205b107 100644 --- a/runtimes/go/appruntime/shared/cloudtrace/logfields.go +++ b/runtimes/go/appruntime/shared/cloudtrace/logfields.go @@ -40,5 +40,23 @@ func StructuredLogFields(req *http.Request) map[string]string { } } + // On Azure, Application Insights correlates logs using the W3C traceparent header. + // If the request carries a traceparent header and Application Insights is configured, + // emit the operation_Id and operation_ParentId fields so Azure Monitor can associate + // the log entry with the distributed trace. + if traceParent := req.Header.Get("traceparent"); traceParent != "" { + if instrKey := AzureInstrumentationKey(); instrKey != "" { + ctx := parseTraceParent(log.Logger, req) + if ctx != nil && !ctx.TraceID.IsZero() { + traceIDHex := fmt.Sprintf("%x", ctx.TraceID[:]) + additionalLogFields["operation_Id"] = traceIDHex + if !ctx.SpanID.IsZero() { + // Application Insights dependency format: |{traceId}.{spanId}. + additionalLogFields["operation_ParentId"] = fmt.Sprintf("|%s.%x.", traceIDHex, ctx.SpanID[:]) + } + } + } + } + return additionalLogFields } diff --git a/runtimes/go/appruntime/shared/nativehist/nativehist.go b/runtimes/go/appruntime/shared/nativehist/nativehist.go index 123c98e999..f617c560ce 100644 --- a/runtimes/go/appruntime/shared/nativehist/nativehist.go +++ b/runtimes/go/appruntime/shared/nativehist/nativehist.go @@ -13,10 +13,33 @@ import ( func New(bucketFactor float64) *Histogram { return &Histogram{ - Schema: pickSchema(bucketFactor), + Schema: pickSchema(bucketFactor), + minBits: math.Float64bits(math.Inf(1)), + maxBits: math.Float64bits(math.Inf(-1)), } } +// Stats is a snapshot of a Histogram's aggregate values. +type Stats struct { + Count uint64 + Sum float64 + Min float64 + Max float64 +} + +// Stats returns a consistent snapshot of the aggregate observation values. +// If no observations have been recorded, Min and Max are both 0. +func (h *Histogram) Stats() Stats { + count := atomic.LoadUint64(&h.Count) + sum := math.Float64frombits(atomic.LoadUint64(&h.sumBits)) + min := math.Float64frombits(atomic.LoadUint64(&h.minBits)) + max := math.Float64frombits(atomic.LoadUint64(&h.maxBits)) + if count == 0 { + min, max = 0, 0 + } + return Stats{Count: count, Sum: sum, Min: min, Max: max} +} + type Histogram struct { // Order in this struct matters for the alignment required by atomic // operations, see http://golang.org/pkg/sync/atomic/#pkg-note-BUG @@ -25,6 +48,17 @@ type Histogram struct { // NumZeroValues counts the number of observations in the zero bucket. NumZeroValues uint64 + // sumBits holds the running sum of observed values encoded as float64 bits. + sumBits uint64 + + // minBits holds the running minimum observed value encoded as float64 bits. + // Initialised to +Inf so the first real observation always wins. + minBits uint64 + + // maxBits holds the running maximum observed value encoded as float64 bits. + // Initialised to -Inf so the first real observation always wins. + maxBits uint64 + // Schema is the Histogram bucket Schema. It's decided on creation. Schema int32 @@ -71,11 +105,46 @@ func (h *Histogram) Observe(v float64) { default: atomic.AddUint64(&h.NumZeroValues, 1) } + + atomic.AddUint64(&h.Count, 1) + + // Update running sum using a CAS loop. + for { + old := atomic.LoadUint64(&h.sumBits) + if atomic.CompareAndSwapUint64(&h.sumBits, old, math.Float64bits(math.Float64frombits(old)+v)) { + break + } + } + + // Update running min using a CAS loop. + for { + old := atomic.LoadUint64(&h.minBits) + if v >= math.Float64frombits(old) { + break + } + if atomic.CompareAndSwapUint64(&h.minBits, old, math.Float64bits(v)) { + break + } + } + + // Update running max using a CAS loop. + for { + old := atomic.LoadUint64(&h.maxBits) + if v <= math.Float64frombits(old) { + break + } + if atomic.CompareAndSwapUint64(&h.maxBits, old, math.Float64bits(v)) { + break + } + } } func (h *Histogram) reset() { atomic.StoreUint64(&h.Count, 0) atomic.StoreUint64(&h.NumZeroValues, 0) + atomic.StoreUint64(&h.sumBits, 0) + atomic.StoreUint64(&h.minBits, math.Float64bits(math.Inf(1))) + atomic.StoreUint64(&h.maxBits, math.Float64bits(math.Inf(-1))) clearSyncMap(&h.PositiveVals) clearSyncMap(&h.NegativeVals) } diff --git a/runtimes/go/go.mod b/runtimes/go/go.mod index 38abc38d73..15b20fab1e 100644 --- a/runtimes/go/go.mod +++ b/runtimes/go/go.mod @@ -7,9 +7,11 @@ require ( cloud.google.com/go/monitoring v1.20.4 cloud.google.com/go/pubsub v1.41.0 cloud.google.com/go/storage v1.41.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 - github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0 + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/DataDog/datadog-api-client-go/v2 v2.9.0 github.com/alicebob/miniredis/v2 v2.23.0 github.com/aws/aws-sdk-go-v2 v1.32.4 @@ -39,9 +41,9 @@ require ( github.com/rs/zerolog v1.31.0 go.encore.dev/platform-sdk v1.1.0 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.25.0 - golang.org/x/net v0.27.0 - golang.org/x/sync v0.8.0 + golang.org/x/crypto v0.47.0 + golang.org/x/net v0.49.0 + golang.org/x/sync v0.19.0 golang.org/x/time v0.6.0 google.golang.org/api v0.191.0 google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f @@ -54,8 +56,10 @@ require ( cloud.google.com/go/auth v0.8.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/iam v1.1.12 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 // indirect + github.com/Azure/go-amqp v1.4.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/DataDog/zstd v1.5.0 // indirect github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect @@ -71,13 +75,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/google/uuid v1.6.0 // indirect @@ -87,7 +90,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.17.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -95,9 +97,8 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/onsi/gomega v1.30.0 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect @@ -106,9 +107,8 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect - nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/runtimes/go/go.sum b/runtimes/go/go.sum index c946185b40..daf90aba2c 100644 --- a/runtimes/go/go.sum +++ b/runtimes/go/go.sum @@ -19,16 +19,30 @@ cloud.google.com/go/pubsub v1.41.0 h1:ZPaM/CvTO6T+1tQOs/jJ4OEMpjtel0PTLV7j1JK+Zr cloud.google.com/go/pubsub v1.41.0/go.mod h1:g+YzC6w/3N91tzG66e2BZtp7WrpBBMXVa3Y9zVoOGpk= cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3 h1:8LoU8N2lIUzkmstvwXvVfniMZlFbesfT2AmA1aqvRr8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.3/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0 h1:ebO2jmZyctLSMBTvjsxZv/Ml3rGsvnJHUImVWotBl7I= -github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.1.0/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= -github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0 h1:kE5kpeiSqu4jcCQ/sWuyggMXJ/pT6oQ99+8hwPmyeJ0= +github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.10.0/go.mod h1:IAN3Z0DMtehoxoQQnfqg1891z1P7GNoDryKtFcAyMBI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0 h1:/g8S6wk65vfC6m3FIxJ+i5QDyN9JWwXI8Hb0Img10hU= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azsecrets v1.4.0/go.mod h1:gpl+q95AzZlKVI3xSoseF9QPrypk0hQqBiJYeB/cR/I= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4= +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/go-amqp v1.4.0 h1:Xj3caqi4comOF/L1Uc5iuBxR/pB6KumejC01YQOqOR4= +github.com/Azure/go-amqp v1.4.0/go.mod h1:vZAogwdrkbyK3Mla8m/CxSc/aKdnTZ4IbPxl51Y5WZE= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-api-client-go/v2 v2.9.0 h1:1Cz3mqj95iqnQPykEovq2p52rrU26XvLC2Fz6hPE+TU= github.com/DataDog/datadog-api-client-go/v2 v2.9.0/go.mod h1:sHt3EuVMN8PSYJu065qwp3pZxCwR3RZP4sJnYwj/ZQY= @@ -83,13 +97,15 @@ github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxY github.com/benbjohnson/clock v1.3.3 h1:g+rSsSaAzhHJYcIQE78hJ3AhyjjtQvleKDjlhdBnIhc= github.com/benbjohnson/clock v1.3.3/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= +github.com/coder/websocket v1.8.13/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -97,8 +113,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -107,29 +121,22 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fmstephe/unsafeutil v1.0.0 h1:hWKjyW7jOL7rfCiBgX61tGy742pZ3C3VpHcGwTAgB2w= github.com/fmstephe/unsafeutil v1.0.0/go.mod h1:00y9QPGpX2A5iB0UmPDtnSpO4c2XsRQu3dQYuGL8+RA= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -139,8 +146,6 @@ github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -174,7 +179,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -187,36 +191,30 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -225,8 +223,8 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -235,8 +233,8 @@ github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4 github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.8.3-0.20221003140808-fcebdb403f4d h1:gNEXs+4IbftZmT6WnAJbBWgbPrjDjqaMfuNeKODqBhc= github.com/rs/cors v1.8.3-0.20221003140808-fcebdb403f4d/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -247,15 +245,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= @@ -283,8 +278,8 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -298,8 +293,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -307,31 +302,28 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -381,7 +373,6 @@ google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWn gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -390,5 +381,3 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/runtimes/go/pubsub/internal/azure/topic_test.go b/runtimes/go/pubsub/internal/azure/topic_test.go new file mode 100644 index 0000000000..f3ccf8dc06 --- /dev/null +++ b/runtimes/go/pubsub/internal/azure/topic_test.go @@ -0,0 +1,231 @@ +package azure + +import ( + "fmt" + "strconv" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus" + + "encore.dev/appruntime/exported/config" +) + +func TestConstants(t *testing.T) { + if RetryCountAttribute != "encore-retry-count" { + t.Errorf("RetryCountAttribute = %q, want %q", RetryCountAttribute, "encore-retry-count") + } + if TargetSubAttribute != "encore-target-sub" { + t.Errorf("TargetSubAttribute = %q, want %q", TargetSubAttribute, "encore-target-sub") + } +} + +func TestManager_ProviderName(t *testing.T) { + mgr := &Manager{_clients: map[string]*azservicebus.Client{}} + got := mgr.ProviderName() + want := "azure" + if got != want { + t.Errorf("ProviderName() = %q, want %q", got, want) + } +} + +func TestManager_Matches(t *testing.T) { + tests := []struct { + name string + cfg *config.PubsubProvider + want bool + }{ + { + name: "nil azure config", + cfg: &config.PubsubProvider{}, + want: false, + }, + { + name: "non-nil azure config", + cfg: &config.PubsubProvider{ + Azure: &config.AzureServiceBusProvider{ + Namespace: "test", + }, + }, + want: true, + }, + { + name: "aws config only", + cfg: &config.PubsubProvider{ + AWS: &config.AWSPubsubProvider{}, + }, + want: false, + }, + { + name: "gcp config only", + cfg: &config.PubsubProvider{ + GCP: &config.GCPPubsubProvider{}, + }, + want: false, + }, + { + name: "multiple providers with azure", + cfg: &config.PubsubProvider{ + Azure: &config.AzureServiceBusProvider{ + Namespace: "test", + }, + AWS: &config.AWSPubsubProvider{}, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mgr := &Manager{_clients: map[string]*azservicebus.Client{}} + got := mgr.Matches(tt.cfg) + if got != tt.want { + t.Errorf("Matches() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewManager(t *testing.T) { + mgr := NewManager(nil) + if mgr._clients == nil { + t.Fatal("_clients map should be initialized") + } + if mgr.ProviderName() != "azure" { + t.Errorf("ProviderName() = %q, want %q", mgr.ProviderName(), "azure") + } +} + +func TestRetryCountParsing(t *testing.T) { + tests := []struct { + name string + value interface{} + wantCount int64 + }{ + { + name: "nil value", + value: nil, + wantCount: 0, + }, + { + name: "integer 0", + value: int64(0), + wantCount: 0, + }, + { + name: "integer 3", + value: int64(3), + wantCount: 3, + }, + { + name: "string 5", + value: "5", + wantCount: 5, + }, + { + name: "invalid string", + value: "not-a-number", + wantCount: 0, + }, + { + name: "large retry count", + value: int64(100), + wantCount: 100, + }, + { + name: "negative count treated as zero", + value: "-1", + wantCount: -1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + props := map[string]interface{}{} + if tt.value != nil { + props[RetryCountAttribute] = tt.value + } + count, _ := strconv.ParseInt(fmt.Sprintf("%v", props[RetryCountAttribute]), 10, 64) + if count != tt.wantCount { + t.Errorf("retry count = %d, want %d", count, tt.wantCount) + } + }) + } +} + +func TestAttributeConversion(t *testing.T) { + applicationProps := map[string]interface{}{ + "string-attr": "hello", + "int-attr": int64(42), + "bool-attr": true, + "float-attr": 3.14, + RetryCountAttribute: int64(2), + "empty-string": "", + "zero-int": int64(0), + "false-bool": false, + } + + attrs := make(map[string]string, len(applicationProps)) + for k, v := range applicationProps { + attrs[k] = fmt.Sprintf("%v", v) + } + + tests := []struct { + key string + want string + }{ + {"string-attr", "hello"}, + {"int-attr", "42"}, + {"bool-attr", "true"}, + {"float-attr", "3.14"}, + {RetryCountAttribute, "2"}, + {"empty-string", ""}, + {"zero-int", "0"}, + {"false-bool", "false"}, + } + + for _, tt := range tests { + t.Run(tt.key, func(t *testing.T) { + got, ok := attrs[tt.key] + if !ok { + t.Errorf("attribute %q not found in converted map", tt.key) + return + } + if got != tt.want { + t.Errorf("attribute %q = %q, want %q", tt.key, got, tt.want) + } + }) + } +} + +func TestDeliveryAttemptCalculation(t *testing.T) { + tests := []struct { + name string + retryCount int64 + wantDelivery int64 + }{ + { + name: "first delivery (no retries)", + retryCount: 0, + wantDelivery: 1, + }, + { + name: "second delivery (one retry)", + retryCount: 1, + wantDelivery: 2, + }, + { + name: "tenth delivery (nine retries)", + retryCount: 9, + wantDelivery: 10, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + deliveryAttempt := tt.retryCount + 1 + if deliveryAttempt != tt.wantDelivery { + t.Errorf("delivery attempt = %d, want %d", deliveryAttempt, tt.wantDelivery) + } + }) + } +} diff --git a/runtimes/go/storage/objects/internal/providers/azblob/azblob_test.go b/runtimes/go/storage/objects/internal/providers/azblob/azblob_test.go new file mode 100644 index 0000000000..ab77a8090f --- /dev/null +++ b/runtimes/go/storage/objects/internal/providers/azblob/azblob_test.go @@ -0,0 +1,382 @@ +//go:build !encore_no_azure + +package azblob + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + qt "github.com/frankban/quicktest" + "github.com/golang/mock/gomock" + + "encore.dev/appruntime/exported/config" + "encore.dev/storage/objects/internal/types" +) + +// ---- parseConnectionString tests ------------------------------------------------------- + +func TestParseConnectionString(t *testing.T) { + tests := []struct { + name string + connStr string + wantName string + wantKey string + }{ + { + name: "full connection string", + connStr: "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey==;EndpointSuffix=core.windows.net", + wantName: "myaccount", + wantKey: "mykey==", + }, + { + name: "minimal connection string", + connStr: "AccountName=foo;AccountKey=bar", + wantName: "foo", + wantKey: "bar", + }, + { + name: "missing account key", + connStr: "AccountName=onlyname", + wantName: "onlyname", + wantKey: "", + }, + { + name: "missing account name", + connStr: "AccountKey=onlykey", + wantName: "", + wantKey: "onlykey", + }, + { + name: "empty string", + connStr: "", + wantName: "", + wantKey: "", + }, + { + name: "key with equals signs (base64 padding)", + connStr: "AccountName=acct;AccountKey=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + wantName: "acct", + wantKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := qt.New(t) + name, key := parseConnectionString(tt.connStr) + c.Assert(name, qt.Equals, tt.wantName) + c.Assert(key, qt.Equals, tt.wantKey) + }) + } +} + +// ---- uploader tests -------------------------------------------------------------------- + +func TestUploader_SingleUpload(t *testing.T) { + c := qt.New(t) + + ctrl := gomock.NewController(c) + client := NewMockblockBlobClient(ctrl) + + const ( + object = "myblob" + contentType = "text/plain" + version = "ver1" + etag = "etag1" + ) + + u := newUploader(client, types.UploadData{ + Ctx: context.Background(), + Object: object, + Attrs: types.UploadAttrs{ContentType: contentType}, + }) + + etagVal := azcore.ETag(etag) + client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(blockblob.UploadResponse{ + VersionID: ptr(version), + ETag: &etagVal, + }, nil) + + content := []byte("hello azure") + n, err := u.Write(content) + c.Assert(n, qt.Equals, len(content)) + c.Assert(err, qt.IsNil) + + attrs, err := u.Complete() + c.Assert(err, qt.IsNil) + c.Assert(attrs, qt.DeepEquals, &types.ObjectAttrs{ + Object: types.CloudObject(object), + Version: version, + ContentType: contentType, + ETag: etag, + Size: int64(len(content)), + }) +} + +func TestUploader_MultipleWrites(t *testing.T) { + c := qt.New(t) + + ctrl := gomock.NewController(c) + client := NewMockblockBlobClient(ctrl) + + const ( + object = "myblob" + contentType = "application/octet-stream" + version = "v2" + etag = "etag2" + ) + + u := newUploader(client, types.UploadData{ + Ctx: context.Background(), + Object: object, + Attrs: types.UploadAttrs{ContentType: contentType}, + }) + + etagVal := azcore.ETag(etag) + client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(blockblob.UploadResponse{ + VersionID: ptr(version), + ETag: &etagVal, + }, nil) + + base := "chunk" + total := strings.Repeat(base, 10) + for i := 0; i < 10; i++ { + n, err := u.Write([]byte(base)) + c.Assert(n, qt.Equals, len(base)) + c.Assert(err, qt.IsNil) + } + + attrs, err := u.Complete() + c.Assert(err, qt.IsNil) + c.Assert(attrs, qt.DeepEquals, &types.ObjectAttrs{ + Object: types.CloudObject(object), + Version: version, + ContentType: contentType, + ETag: etag, + Size: int64(len(total)), + }) +} + +func TestUploader_MultipartUpload(t *testing.T) { + c := qt.New(t) + + // Use a small buffer so writes spill across buffers and trigger multipart. + withBufSize(c, 10) + + ctrl := gomock.NewController(c) + client := NewMockblockBlobClient(ctrl) + + const ( + object = "bigblob" + contentType = "text/plain" + version = "v3" + etag = "etag3" + ) + + u := newUploader(client, types.UploadData{ + Ctx: context.Background(), + Object: object, + Attrs: types.UploadAttrs{ContentType: contentType}, + }) + + // Writing "abcdefghijklm" × 3 (39 bytes) with bufSize=10 produces 4 blocks: + // "abcdefghij" / "klmabcdefg" / "hijklmabcd" / "efghijklm" + client.EXPECT().StageBlock(gomock.Any(), blockIDForPart(0), &blockBodyMatcher{"abcdefghij"}, gomock.Any()).Return(blockblob.StageBlockResponse{}, nil) + client.EXPECT().StageBlock(gomock.Any(), blockIDForPart(1), &blockBodyMatcher{"klmabcdefg"}, gomock.Any()).Return(blockblob.StageBlockResponse{}, nil) + client.EXPECT().StageBlock(gomock.Any(), blockIDForPart(2), &blockBodyMatcher{"hijklmabcd"}, gomock.Any()).Return(blockblob.StageBlockResponse{}, nil) + client.EXPECT().StageBlock(gomock.Any(), blockIDForPart(3), &blockBodyMatcher{"efghijklm"}, gomock.Any()).Return(blockblob.StageBlockResponse{}, nil) + + etagVal := azcore.ETag(etag) + client.EXPECT().CommitBlockList(gomock.Any(), gomock.Any(), gomock.Any()).Return(blockblob.CommitBlockListResponse{ + VersionID: ptr(version), + ETag: &etagVal, + }, nil) + + base := "abcdefghijklm" + total := strings.Repeat(base, 3) + for i := 0; i < 3; i++ { + n, err := u.Write([]byte(base)) + c.Assert(n, qt.Equals, len(base)) + c.Assert(err, qt.IsNil) + } + + attrs, err := u.Complete() + c.Assert(err, qt.IsNil) + c.Assert(attrs, qt.DeepEquals, &types.ObjectAttrs{ + Object: types.CloudObject(object), + Version: version, + ContentType: contentType, + ETag: etag, + Size: int64(len(total)), + }) +} + +func TestUploader_EmptyUpload(t *testing.T) { + c := qt.New(t) + + ctrl := gomock.NewController(c) + client := NewMockblockBlobClient(ctrl) + + etagVal := azcore.ETag("e") + client.EXPECT().Upload(gomock.Any(), gomock.Any(), gomock.Any()).Return(blockblob.UploadResponse{ + ETag: &etagVal, + }, nil) + + u := newUploader(client, types.UploadData{ + Ctx: context.Background(), + Object: "empty", + Attrs: types.UploadAttrs{}, + }) + + attrs, err := u.Complete() + c.Assert(err, qt.IsNil) + c.Assert(attrs.Size, qt.Equals, int64(0)) +} + +// withBufSize overrides the package-level bufSize and resets the pool so that +// newly allocated buffers use the new size. +func withBufSize(c *qt.C, n int) { + origSize := bufSize + origPool := bufPool + bufSize = n + bufPool = sync.Pool{New: func() any { return &buffer{buf: make([]byte, bufSize)} }} + c.Cleanup(func() { + bufSize = origSize + bufPool = origPool + }) +} + +// blockBodyMatcher is a gomock.Matcher that reads an io.ReadSeekCloser and +// compares its content to an expected string. +type blockBodyMatcher struct { + data string +} + +func (m *blockBodyMatcher) Matches(x interface{}) bool { + body, ok := x.(io.ReadSeekCloser) + if !ok { + return false + } + got, err := io.ReadAll(body) + if err != nil { + return false + } + // Reset so subsequent reads by the production code work. + _, _ = body.Seek(0, io.SeekStart) + return string(got) == m.data +} + +func (m *blockBodyMatcher) String() string { + return fmt.Sprintf("body == %q", m.data) +} + +// ---- SAS URL tests --------------------------------------------------------------------- + +// testSharedKey creates a SharedKeyCredential for testing using a 64-byte zero key. +func testSharedKey(t *testing.T, accountName string) *azblob.SharedKeyCredential { + t.Helper() + key := base64.StdEncoding.EncodeToString(make([]byte, 64)) + cred, err := azblob.NewSharedKeyCredential(accountName, key) + if err != nil { + t.Fatalf("create test SharedKeyCredential: %v", err) + } + return cred +} + +func TestSignedUploadURL(t *testing.T) { + c := qt.New(t) + + const accountName = "testaccount" + const containerName = "mycontainer" + const blobName = "path/to/myblob.txt" + + b := &bucket{ + sharedKey: testSharedKey(t, accountName), + accountName: accountName, + cfg: &config.Bucket{CloudName: containerName}, + } + + url, err := b.SignedUploadURL(types.UploadURLData{ + Ctx: context.Background(), + Object: types.CloudObject(blobName), + TTL: time.Hour, + }) + c.Assert(err, qt.IsNil) + + expectedPrefix := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?", accountName, containerName, blobName) + c.Assert(strings.HasPrefix(url, expectedPrefix), qt.IsTrue, + qt.Commentf("URL %q should start with %q", url, expectedPrefix)) + c.Assert(strings.Contains(url, "sp="), qt.IsTrue, + qt.Commentf("URL should contain SAS permissions param; got %q", url)) + c.Assert(strings.Contains(url, "sig="), qt.IsTrue, + qt.Commentf("URL should contain SAS signature; got %q", url)) + c.Assert(strings.Contains(url, "spr=https"), qt.IsTrue, + qt.Commentf("URL should require HTTPS; got %q", url)) +} + +func TestSignedDownloadURL(t *testing.T) { + c := qt.New(t) + + const accountName = "testaccount" + const containerName = "mycontainer" + const blobName = "path/to/file.bin" + + b := &bucket{ + sharedKey: testSharedKey(t, accountName), + accountName: accountName, + cfg: &config.Bucket{CloudName: containerName}, + } + + url, err := b.SignedDownloadURL(types.DownloadURLData{ + Ctx: context.Background(), + Object: types.CloudObject(blobName), + TTL: 15 * time.Minute, + }) + c.Assert(err, qt.IsNil) + + expectedPrefix := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?", accountName, containerName, blobName) + c.Assert(strings.HasPrefix(url, expectedPrefix), qt.IsTrue, + qt.Commentf("URL %q should start with %q", url, expectedPrefix)) + c.Assert(strings.Contains(url, "sp="), qt.IsTrue, + qt.Commentf("URL should contain SAS permissions param; got %q", url)) + c.Assert(strings.Contains(url, "sig="), qt.IsTrue, + qt.Commentf("URL should contain SAS signature; got %q", url)) +} + +func TestSignedURL_NoSharedKey(t *testing.T) { + c := qt.New(t) + + b := &bucket{ + sharedKey: nil, + accountName: "account", + cfg: &config.Bucket{CloudName: "container"}, + } + + _, err := b.SignedUploadURL(types.UploadURLData{ + Ctx: context.Background(), + Object: "blob", + TTL: time.Hour, + }) + c.Assert(err, qt.Not(qt.IsNil)) + + _, err = b.SignedDownloadURL(types.DownloadURLData{ + Ctx: context.Background(), + Object: "blob", + TTL: time.Hour, + }) + c.Assert(err, qt.Not(qt.IsNil)) +} + +// blockBodyMatcher uses bytes.NewReader so we can seek; make sure the body +// is a real bytes.Reader-backed seeker. +var _ = (*bytes.Reader)(nil) diff --git a/runtimes/go/storage/objects/internal/providers/azblob/bucket.go b/runtimes/go/storage/objects/internal/providers/azblob/bucket.go new file mode 100644 index 0000000000..438219868f --- /dev/null +++ b/runtimes/go/storage/objects/internal/providers/azblob/bucket.go @@ -0,0 +1,317 @@ +//go:build !encore_no_azure + +package azblob + +import ( + "context" + "fmt" + "iter" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + + "encore.dev/appruntime/exported/config" + "encore.dev/storage/objects/internal/types" +) + +// Manager manages Azure Blob Storage bucket clients. +// +// NOTE: Azure Blob Storage proto/config support (AzureBlobBucketProvider) was +// added to config.go but does not yet exist in infra.proto. When the proto +// definition is added, the config parsing layer will need to be updated to +// populate AzureBlobBucketProvider from the proto message. +type Manager struct { + ctx context.Context + runtime *config.Runtime + clients map[*config.BucketProvider]*clientState +} + +type clientState struct { + serviceClient *azblob.Client + sharedKey *azblob.SharedKeyCredential // nil when using managed identity + accountName string +} + +func NewManager(ctx context.Context, runtime *config.Runtime) *Manager { + return &Manager{ + ctx: ctx, + runtime: runtime, + clients: make(map[*config.BucketProvider]*clientState), + } +} + +type bucket struct { + containerClient *container.Client + sharedKey *azblob.SharedKeyCredential // nil when using managed identity + accountName string + cfg *config.Bucket +} + +func (mgr *Manager) ProviderName() string { return "azure-blob" } + +func (mgr *Manager) Matches(cfg *config.BucketProvider) bool { + return cfg.AzureBlob != nil +} + +func (mgr *Manager) NewBucket(provider *config.BucketProvider, runtimeCfg *config.Bucket) types.BucketImpl { + state := mgr.clientForProvider(provider) + containerClient := state.serviceClient.ServiceClient().NewContainerClient(runtimeCfg.CloudName) + return &bucket{ + containerClient: containerClient, + sharedKey: state.sharedKey, + accountName: state.accountName, + cfg: runtimeCfg, + } +} + +func (b *bucket) Download(data types.DownloadData) (types.Downloader, error) { + blobClient := b.containerClient.NewBlockBlobClient(data.Object.String()) + if data.Version != "" { + var err error + blobClient, err = blobClient.WithVersionID(data.Version) + if err != nil { + return nil, err + } + } + resp, err := blobClient.DownloadStream(data.Ctx, nil) + if err != nil { + return nil, mapErr(err) + } + return resp.Body, nil +} + +func (b *bucket) Upload(data types.UploadData) (types.Uploader, error) { + blobClient := b.containerClient.NewBlockBlobClient(data.Object.String()) + return newUploader(blobClient, data), nil +} + +func (b *bucket) List(data types.ListData) iter.Seq2[*types.ListEntry, error] { + return func(yield func(*types.ListEntry, error) bool) { + var n int64 + pager := b.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Prefix: ptrOrNil(data.Prefix), + }) + for pager.More() { + if err := data.Ctx.Err(); err != nil { + yield(nil, err) + return + } + resp, err := pager.NextPage(data.Ctx) + if err != nil { + yield(nil, mapErr(err)) + return + } + for _, item := range resp.Segment.BlobItems { + if data.Limit != nil && n >= *data.Limit { + return + } + n++ + entry := &types.ListEntry{ + Object: types.CloudObject(valOrZero(item.Name)), + Size: valOrZero(item.Properties.ContentLength), + ETag: string(valOrZero(item.Properties.ETag)), + } + if !yield(entry, nil) { + return + } + } + } + } +} + +func (b *bucket) Remove(data types.RemoveData) error { + blobClient := b.containerClient.NewBlockBlobClient(data.Object.String()) + if data.Version != "" { + var err error + blobClient, err = blobClient.WithVersionID(data.Version) + if err != nil { + return err + } + } + _, err := blobClient.Delete(data.Ctx, nil) + return mapErr(err) +} + +func (b *bucket) Attrs(data types.AttrsData) (*types.ObjectAttrs, error) { + blobClient := b.containerClient.NewBlockBlobClient(data.Object.String()) + if data.Version != "" { + var err error + blobClient, err = blobClient.WithVersionID(data.Version) + if err != nil { + return nil, err + } + } + resp, err := blobClient.GetProperties(data.Ctx, nil) + if err != nil { + return nil, mapErr(err) + } + return &types.ObjectAttrs{ + Object: data.Object, + Version: valOrZero(resp.VersionID), + ContentType: valOrZero(resp.ContentType), + Size: valOrZero(resp.ContentLength), + ETag: string(valOrZero(resp.ETag)), + }, nil +} + +func (b *bucket) SignedUploadURL(data types.UploadURLData) (string, error) { + if b.sharedKey == nil { + return "", fmt.Errorf("azure blob: signed URLs require SharedKey credentials; provide a storage_key or connection_string") + } + blobName := data.Object.String() + perms := sas.BlobPermissions{Write: true, Create: true} + sasParams, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + StartTime: time.Now().UTC().Add(-10 * time.Second), // small buffer for clock skew + ExpiryTime: time.Now().UTC().Add(data.TTL), + Permissions: perms.String(), + ContainerName: b.cfg.CloudName, + BlobName: blobName, + }.SignWithSharedKey(b.sharedKey) + if err != nil { + return "", mapErr(err) + } + return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?%s", + b.accountName, b.cfg.CloudName, blobName, sasParams.Encode()), nil +} + +func (b *bucket) SignedDownloadURL(data types.DownloadURLData) (string, error) { + if b.sharedKey == nil { + return "", fmt.Errorf("azure blob: signed URLs require SharedKey credentials; provide a storage_key or connection_string") + } + blobName := data.Object.String() + perms := sas.BlobPermissions{Read: true} + sasParams, err := sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, + StartTime: time.Now().UTC().Add(-10 * time.Second), // small buffer for clock skew + ExpiryTime: time.Now().UTC().Add(data.TTL), + Permissions: perms.String(), + ContainerName: b.cfg.CloudName, + BlobName: blobName, + }.SignWithSharedKey(b.sharedKey) + if err != nil { + return "", mapErr(err) + } + return fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s?%s", + b.accountName, b.cfg.CloudName, blobName, sasParams.Encode()), nil +} + +func (mgr *Manager) clientForProvider(prov *config.BucketProvider) *clientState { + if state, ok := mgr.clients[prov]; ok { + return state + } + + cfg := prov.AzureBlob + serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.StorageAccount) + + var ( + client *azblob.Client + sharedKey *azblob.SharedKeyCredential + err error + ) + + switch { + case cfg.ConnectionString != nil: + // Connection string auth: create the service client from the connection string. + client, err = azblob.NewClientFromConnectionString(*cfg.ConnectionString, nil) + if err != nil { + panic(fmt.Sprintf("azure blob: failed to create client from connection string: %v", err)) + } + // Try to extract AccountName + AccountKey from the connection string so we + // can generate SAS URLs. Connection strings look like: + // DefaultEndpointsProtocol=https;AccountName=xxx;AccountKey=yyy==;EndpointSuffix=... + if accountName, accountKey := parseConnectionString(*cfg.ConnectionString); accountName != "" && accountKey != "" { + sharedKey, err = azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + panic(fmt.Sprintf("azure blob: failed to create shared key credential from connection string: %v", err)) + } + cfg.StorageAccount = accountName // ensure accountName is set for SAS URL generation + } + + case cfg.StorageKey != nil: + // Explicit SharedKey authentication. + sharedKey, err = azblob.NewSharedKeyCredential(cfg.StorageAccount, *cfg.StorageKey) + if err != nil { + panic(fmt.Sprintf("azure blob: failed to create shared key credential: %v", err)) + } + client, err = azblob.NewClientWithSharedKeyCredential(serviceURL, sharedKey, nil) + if err != nil { + panic(fmt.Sprintf("azure blob: failed to create Azure Blob client with shared key: %v", err)) + } + + default: + // No explicit credentials: use DefaultAzureCredential (managed identity, env vars, etc.). + cred, credErr := azidentity.NewDefaultAzureCredential(nil) + if credErr != nil { + panic(fmt.Sprintf("azure blob: failed to create default Azure credential: %v", credErr)) + } + client, err = azblob.NewClient(serviceURL, cred, nil) + if err != nil { + panic(fmt.Sprintf("azure blob: failed to create Azure Blob client: %v", err)) + } + } + + state := &clientState{ + serviceClient: client, + sharedKey: sharedKey, + accountName: cfg.StorageAccount, + } + mgr.clients[prov] = state + return state +} + +// parseConnectionString extracts the AccountName and AccountKey from an Azure +// Blob Storage connection string of the form: +// +// DefaultEndpointsProtocol=https;AccountName=;AccountKey=;... +func parseConnectionString(connStr string) (accountName, accountKey string) { + for _, segment := range strings.Split(connStr, ";") { + kv := strings.SplitN(segment, "=", 2) + if len(kv) != 2 { + continue + } + switch kv[0] { + case "AccountName": + accountName = kv[1] + case "AccountKey": + accountKey = kv[1] + } + } + return +} + +func mapErr(err error) error { + if err == nil { + return nil + } + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return types.ErrObjectNotExist + } + if bloberror.HasCode(err, bloberror.ConditionNotMet) { + return types.ErrPreconditionFailed + } + return err +} + +func ptrOrNil[T comparable](v T) *T { + var zero T + if v == zero { + return nil + } + return &v +} + +func valOrZero[T any](p *T) T { + if p == nil { + var zero T + return zero + } + return *p +} + +func ptr[T any](v T) *T { return &v } diff --git a/runtimes/go/storage/objects/internal/providers/azblob/mock_blockblob_client_test.go b/runtimes/go/storage/objects/internal/providers/azblob/mock_blockblob_client_test.go new file mode 100644 index 0000000000..49fd7f8f74 --- /dev/null +++ b/runtimes/go/storage/objects/internal/providers/azblob/mock_blockblob_client_test.go @@ -0,0 +1,84 @@ +//go:build !encore_no_azure + +package azblob + +import ( + "context" + "io" + "reflect" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/golang/mock/gomock" +) + +// MockblockBlobClient is a hand-written mock of the blockBlobClient interface, +// following the same pattern as the generated S3 mock. +type MockblockBlobClient struct { + ctrl *gomock.Controller + recorder *MockblockBlobClientMockRecorder +} + +// MockblockBlobClientMockRecorder records expected calls. +type MockblockBlobClientMockRecorder struct { + mock *MockblockBlobClient +} + +// NewMockblockBlobClient creates a new mock instance. +func NewMockblockBlobClient(ctrl *gomock.Controller) *MockblockBlobClient { + mock := &MockblockBlobClient{ctrl: ctrl} + mock.recorder = &MockblockBlobClientMockRecorder{mock} + return mock +} + +// EXPECT returns the recorder for expected calls. +func (m *MockblockBlobClient) EXPECT() *MockblockBlobClientMockRecorder { + return m.recorder +} + +// Upload mocks blockBlobClient.Upload. +func (m *MockblockBlobClient) Upload(ctx context.Context, body io.ReadSeekCloser, options *blockblob.UploadOptions) (blockblob.UploadResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upload", ctx, body, options) + ret0, _ := ret[0].(blockblob.UploadResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upload records an expected Upload call. +func (mr *MockblockBlobClientMockRecorder) Upload(ctx, body, options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upload", + reflect.TypeOf((*MockblockBlobClient)(nil).Upload), ctx, body, options) +} + +// StageBlock mocks blockBlobClient.StageBlock. +func (m *MockblockBlobClient) StageBlock(ctx context.Context, base64BlockID string, body io.ReadSeekCloser, options *blockblob.StageBlockOptions) (blockblob.StageBlockResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StageBlock", ctx, base64BlockID, body, options) + ret0, _ := ret[0].(blockblob.StageBlockResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StageBlock records an expected StageBlock call. +func (mr *MockblockBlobClientMockRecorder) StageBlock(ctx, base64BlockID, body, options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StageBlock", + reflect.TypeOf((*MockblockBlobClient)(nil).StageBlock), ctx, base64BlockID, body, options) +} + +// CommitBlockList mocks blockBlobClient.CommitBlockList. +func (m *MockblockBlobClient) CommitBlockList(ctx context.Context, base64BlockIDs []string, options *blockblob.CommitBlockListOptions) (blockblob.CommitBlockListResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CommitBlockList", ctx, base64BlockIDs, options) + ret0, _ := ret[0].(blockblob.CommitBlockListResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CommitBlockList records an expected CommitBlockList call. +func (mr *MockblockBlobClientMockRecorder) CommitBlockList(ctx, base64BlockIDs, options interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitBlockList", + reflect.TypeOf((*MockblockBlobClient)(nil).CommitBlockList), ctx, base64BlockIDs, options) +} diff --git a/runtimes/go/storage/objects/internal/providers/azblob/uploader.go b/runtimes/go/storage/objects/internal/providers/azblob/uploader.go new file mode 100644 index 0000000000..25ea35b35a --- /dev/null +++ b/runtimes/go/storage/objects/internal/providers/azblob/uploader.go @@ -0,0 +1,281 @@ +//go:build !encore_no_azure + +package azblob + +import ( +"bytes" +"context" +"encoding/base64" +"encoding/binary" +"errors" +"io" +"sync" + +"github.com/Azure/azure-sdk-for-go/sdk/azcore" +"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" +"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" +"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" +"golang.org/x/sync/errgroup" + +"encore.dev/storage/objects/internal/types" +) + +// uploader implements types.Uploader for Azure Block Blobs. +// +// Small uploads (data fits in the first 10 MiB buffer) use a single +// blockblob.Upload call. Larger uploads use staged blocks +// (StageBlock x N -> CommitBlockList), which mirrors the S3 multipart pattern. +type uploader struct { +client blockBlobClient +data types.UploadData +ctx context.Context +out chan uploadEvent + +init sync.Once +done chan struct{} +attrs *types.ObjectAttrs +err error + +curr *buffer +} + +type uploadEvent struct { +data *buffer +abort error +done bool +} + +type buffer struct { +buf []byte +n int +} + +func newUploader(client blockBlobClient, data types.UploadData) *uploader { +return &uploader{ +client: client, +ctx: data.Ctx, +data: data, +out: make(chan uploadEvent, 10), +done: make(chan struct{}), +} +} + +func (u *uploader) Write(p []byte) (n int, err error) { +u.initUpload() +for len(p) > 0 { +curr := u.curr +if curr == nil { +curr = getBuf() +} + +copied := copy(curr.buf[curr.n:], p) +n += copied +curr.n += copied + +if copied < len(p) { +p = p[copied:] +select { +case u.out <- uploadEvent{data: curr}: +case <-u.done: +return n, u.err +} +u.curr, curr = nil, nil +} else { +u.curr = curr +return n, nil +} +} +return n, nil +} + +func (u *uploader) Complete() (*types.ObjectAttrs, error) { +u.initUpload() +if curr := u.curr; curr != nil && curr.n > 0 { +select { +case u.out <- uploadEvent{data: curr, done: true}: +case <-u.done: +} +u.curr = nil +} else { +select { +case u.out <- uploadEvent{done: true}: +case <-u.done: +} +} +<-u.done +return u.attrs, u.err +} + +func (u *uploader) Abort(err error) { +u.initUpload() +if err == nil { +err = errors.New("upload aborted") +} +select { +case u.out <- uploadEvent{abort: err}: +case <-u.done: +} +} + +func (u *uploader) initUpload() { +u.init.Do(func() { +go func() { +defer close(u.done) +attrs, err := u.doUpload() +u.attrs, u.err = attrs, mapErr(err) +}() +}) +} + +func (u *uploader) doUpload() (*types.ObjectAttrs, error) { +ev := <-u.out +if ev.abort != nil { +return nil, ev.abort +} else if ev.done { +// All data fits in the first buffer (or there is no data): single-part upload. +var buf []byte +if ev.data != nil { +buf = ev.data.buf[:ev.data.n] +} +return u.singlePartUpload(buf) +} +return u.multiPartUpload(ev.data) +} + +// blockBlobClient is the subset of blockblob.Client used by the uploader. +// The interface enables unit testing without a real Azure endpoint. +type blockBlobClient interface { +Upload(ctx context.Context, body io.ReadSeekCloser, options *blockblob.UploadOptions) (blockblob.UploadResponse, error) +StageBlock(ctx context.Context, base64BlockID string, body io.ReadSeekCloser, options *blockblob.StageBlockOptions) (blockblob.StageBlockResponse, error) +CommitBlockList(ctx context.Context, base64BlockIDs []string, options *blockblob.CommitBlockListOptions) (blockblob.CommitBlockListResponse, error) +} + +func (u *uploader) singlePartUpload(buf []byte) (*types.ObjectAttrs, error) { +opts := &blockblob.UploadOptions{} +if u.data.Pre.NotExists { +etagAny := azcore.ETagAny +opts.AccessConditions = &blob.AccessConditions{ +ModifiedAccessConditions: &blob.ModifiedAccessConditions{ +IfNoneMatch: &etagAny, +}, +} +} +if ct := u.data.Attrs.ContentType; ct != "" { +opts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: ptr(ct)} +} + +resp, err := u.client.Upload(u.ctx, streaming.NopCloser(bytes.NewReader(buf)), opts) +if err != nil { +return nil, err +} +return &types.ObjectAttrs{ +Object: u.data.Object, +Version: valOrZero(resp.VersionID), +ContentType: u.data.Attrs.ContentType, +Size: int64(len(buf)), +ETag: string(valOrZero(resp.ETag)), +}, nil +} + +func (u *uploader) multiPartUpload(initial *buffer) (attrs *types.ObjectAttrs, err error) { +g, groupCtx := errgroup.WithContext(u.ctx) +var ( +blockIDs []string +totalSize int64 +part int32 +) + +// stageBlock is called sequentially from the event loop below; the errgroup +// goroutines only perform the network upload, so blockIDs slice ordering is safe. +stageBlock := func(buf *buffer) { +if buf == nil { +return +} +totalSize += int64(buf.n) +blockID := blockIDForPart(part) +part++ +blockIDs = append(blockIDs, blockID) + +g.Go(func() error { +data := buf.buf[:buf.n] +defer putBuf(buf) +_, stageErr := u.client.StageBlock(groupCtx, blockID, streaming.NopCloser(bytes.NewReader(data)), nil) +return stageErr +}) +} + +stageBlock(initial) +for { +ev := <-u.out +if ev.abort != nil { +// Uncommitted blocks in Azure expire automatically; no explicit abort needed. +_ = g.Wait() +return nil, ev.abort +} +if ev.data != nil { +stageBlock(ev.data) +} +if ev.done { +break +} +} + +if err = g.Wait(); err != nil { +return nil, err +} + +commitOpts := &blockblob.CommitBlockListOptions{} +if u.data.Pre.NotExists { +etagAny := azcore.ETagAny +commitOpts.AccessConditions = &blob.AccessConditions{ +ModifiedAccessConditions: &blob.ModifiedAccessConditions{ +IfNoneMatch: &etagAny, +}, +} +} +if ct := u.data.Attrs.ContentType; ct != "" { +commitOpts.HTTPHeaders = &blob.HTTPHeaders{BlobContentType: ptr(ct)} +} + +commitResp, err := u.client.CommitBlockList(u.ctx, blockIDs, commitOpts) +if err != nil { +return nil, err +} +return &types.ObjectAttrs{ +Object: u.data.Object, +Version: valOrZero(commitResp.VersionID), +ContentType: u.data.Attrs.ContentType, +Size: totalSize, +ETag: string(valOrZero(commitResp.ETag)), +}, nil +} + +// blockIDForPart returns a fixed-length base64-encoded block ID for the given +// part index. Azure requires all block IDs for a blob to have the same byte +// length before base64 encoding; we use a 4-byte big-endian representation. +func blockIDForPart(n int32) string { +b := make([]byte, 4) +binary.BigEndian.PutUint32(b, uint32(n)) +return base64.StdEncoding.EncodeToString(b) +} + +// bufSize is the target buffer size for each upload part. +// Variable for testing. Azure supports blocks up to 100 MiB; 10 MiB matches +// the S3 provider default. +var bufSize = 10 * 1024 * 1024 + +var bufPool = sync.Pool{ +New: func() any { +return &buffer{buf: make([]byte, bufSize)} +}, +} + +func getBuf() *buffer { +buf := bufPool.Get().(*buffer) +buf.n = 0 +return buf +} + +func putBuf(buf *buffer) { +bufPool.Put(buf) +} diff --git a/runtimes/go/storage/objects/provider_azblob.go b/runtimes/go/storage/objects/provider_azblob.go new file mode 100644 index 0000000000..e29e746b4a --- /dev/null +++ b/runtimes/go/storage/objects/provider_azblob.go @@ -0,0 +1,16 @@ +//go:build !encore_no_azure + +package objects + +import ( + "context" + + "encore.dev/appruntime/exported/config" + "encore.dev/storage/objects/internal/providers/azblob" +) + +func init() { + registerProvider(func(ctx context.Context, runtimeCfg *config.Runtime) provider { + return azblob.NewManager(ctx, runtimeCfg) + }) +}