feat(hydro_test): add Kafka example with producer/consumer module

- Add hydro_test/src/kafka/mod.rs with kafka_producer, kafka_consumer,
  dest_kafka, and setup_topic helpers following the SQS PR #2746 pattern
- Complete hydro_test/examples/kafka.rs: leader produces 1M financial
  transactions, consumer cluster computes per-account balances
- Add 'kafka' feature flag gating rdkafka and futures-util as optional
  deps
- Add llvm-tools component to rust-toolchain.toml for rust-lld

Author: akainth015

.github/workflows/ci.yml

@@ -64,6 +64,10 @@ jobs:
       - if: matrix.rust_release == 'latest-stable' && matrix.os == 'ubuntu-latest'
         uses: ./.github/actions/use-sccache
 
+      - name: Install libcurl dev (needed by rdkafka-sys)
+        if: runner.os == 'Linux'
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq libcurl4-openssl-dev > /dev/null
+
       - run: cargo clippy --all-targets -- -D warnings
       - run: cargo clippy --all-targets --no-default-features -- -D warnings
       - run: cargo clippy --all-targets --all-features -- -D warnings

.github/workflows/docs.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Install nightly Rust channel
         run: rustup toolchain add nightly
 
+      - name: Install libcurl dev (needed by rdkafka-sys)
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq libcurl4-openssl-dev > /dev/null
+
       - env:
           RUSTDOCFLAGS: --cfg docsrs -Dwarnings
         run: cargo +nightly doc --no-deps --all-features

Cargo.lock

@@ -1,5 +1,7 @@
 set -e
 
+sudo apt-get update -qq && sudo apt-get install -y -qq libcurl4-openssl-dev > /dev/null
+
 echo "========================================="
 echo "Step 1/7: Downloading LLVM..."
 echo "========================================="

build_docs.bash

@@ -47,6 +47,16 @@ pub struct NetworkResources {
     security_group: String,
 }
 
+impl NetworkResources {
+    pub fn new(vpc: String, subnet: String, security_group: String) -> Self {
+        Self {
+            vpc,
+            subnet,
+            security_group,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct AwsNetwork {
     pub region: String,
@@ -910,6 +920,10 @@ echo -e "{cwa_config_esc}" > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwa
                     "associate_public_ip_address": true,
                     "iam_instance_profile": iam_instance_profile_ref, // May be `None`.
                     "user_data": user_data_script, // May be `None`.
+                    "metadata_options": {
+                        "http_tokens": "required",
+                        "http_endpoint": "enabled"
+                    },
                     "tags": {
                         "Name": instance_name
                     }

hydro_deploy/core/src/aws.rs

@@ -29,7 +29,7 @@ pub mod azure;
 pub use azure::AzureHost;
 
 pub mod aws;
-pub use aws::{AwsEc2Host, AwsNetwork};
+pub use aws::{AwsEc2Host, AwsNetwork, NetworkResources};
 
 pub mod rust_crate;
 pub use rust_crate::RustCrate;

hydro_deploy/core/src/lib.rs

@@ -12,6 +12,11 @@ docker = ["hydro_lang/docker_deploy"]
 ecs = ["hydro_lang/ecs_deploy"]
 maelstrom = ["hydro_lang/maelstrom"]
 stageleft_macro_entrypoint = ["hydro_lang/stageleft_macro_entrypoint"]
+kafka = ["dep:rdkafka", "dep:futures-util"]
+
+[[example]]
+name = "kafka"
+required-features = ["kafka"]
 
 [dependencies]
 hydro_lang = { path = "../hydro_lang", version = "^0.15.0" }
@@ -33,6 +38,8 @@ bytes = "1.1.0"
 # https://github.com/GitoxideLabs/cargo-smart-release/issues/36
 example_test = { path = "../example_test", version = "^0.0.0", optional = true }
 hydro_build_utils = { path = "../hydro_build_utils", version = "^0.0.1", optional = true }
+rdkafka = { version = "0.39.0", optional = true, features = ["cmake-build", "ssl-vendored"] }
+futures-util = { version = "0.3.0", default-features = false, features = ["alloc"], optional = true }
 
 [build-dependencies]
 stageleft_tool.workspace = true