diff --git a/Cargo.lock b/Cargo.lock index 34f3c91a..262ff16d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -869,7 +869,6 @@ dependencies = [ "kube", "log", "maplit", - "openssh", "reqwest", "resource-agent", "serde", @@ -2299,15 +2298,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "non-zero-byte-slice" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89daa1daa11c9df05d1181bcd0936d8066f8543144d77b09808eb78d65e38024" -dependencies = [ - "serde", -] - [[package]] name = "nonzero_ext" version = "0.3.0" @@ -2364,50 +2354,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "openssh" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330f4b61092456dc0aaa0cf9a205d956cae07d8127a69ffeff6760a72549c77f" -dependencies = [ - "libc", - "once_cell", - "openssh-mux-client", - "shell-escape", - "tempfile", - "thiserror", - "tokio", - "tokio-pipe", -] - -[[package]] -name = "openssh-mux-client" -version = "0.17.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d091692d0a7d9035a3cdc5f8c3572abe401e64e30eac5404e517fb54e554c9bc" -dependencies = [ - "cfg-if", - "non-zero-byte-slice", - "once_cell", - "openssh-mux-client-error", - "sendfd", - "serde", - "ssh_format", - "tokio", - "tokio-io-utility", - "typed-builder", -] - -[[package]] -name = "openssh-mux-client-error" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "015d49e592f4d2a456033e6ec48036588e8e58c8908424b1bc40994de58ae648" -dependencies = [ - "ssh_format_error", - "thiserror", -] - [[package]] name = "openssl-probe" version = "0.1.6" @@ -3144,16 +3090,6 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" -[[package]] -name = "sendfd" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604b71b8fc267e13bb3023a2c901126c8f349393666a6d98ac1ae5729b701798" -dependencies = [ - "libc", - "tokio", -] - [[package]] name = "serde" version = "1.0.219" @@ -3263,12 +3199,6 @@ dependencies = [ "digest", ] -[[package]] -name = "shell-escape" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" - [[package]] name = "shlex" version = "1.3.0" @@ -3332,25 +3262,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ssh_format" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ab31081d1c9097c327ec23550858cb5ffb4af6b866c1ef4d728455f01f3304" -dependencies = [ - "serde", - "ssh_format_error", -] - -[[package]] -name = "ssh_format_error" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3c6519de7ca611f71ef7e8a56eb57aa1c818fecb5242d0a0f39c83776c210c" -dependencies = [ - "serde", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3668,15 +3579,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-io-utility" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d672654d175710e52c7c41f6aec77c62b3c0954e2a7ebce9049d1e94ed7c263" -dependencies = [ - "tokio", -] - [[package]] name = "tokio-macros" version = "2.5.0" @@ -3688,16 +3590,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "tokio-pipe" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f213a84bffbd61b8fa0ba8a044b4bbe35d471d0b518867181e82bd5c15542784" -dependencies = [ - "libc", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -3911,26 +3803,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "typed-builder" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "typed-path" version = "0.9.3" diff --git a/Dockerfile b/Dockerfile index a096abdd..a56220b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -176,31 +176,6 @@ COPY --from=build-src /usr/share/licenses/testsys /licenses/testsys CMD dockerd --storage-driver vfs &>/dev/null & ./vsphere-k8s-cluster-resource-agent -# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -# Builds the Metal K8s cluster resource agent image -FROM public.ecr.aws/amazonlinux/amazonlinux:2 as metal-k8s-cluster-resource-agent - -RUN yum install -y \ - openssh-clients \ - tar \ - && yum clean all -RUN amazon-linux-extras install -y docker - -# Copy eksctl -COPY --from=tools /eksctl /usr/bin/eksctl -COPY --from=tools /licenses/eksctl /licenses/eksctl - -# Copy kubectl -COPY --from=tools /kubectl /usr/local/bin/kubectl -COPY --from=tools /licenses/kubernetes /licenses/kubernetes - -# Copy binary -COPY --from=build-src /src/bottlerocket/agents/bin/metal-k8s-cluster-resource-agent ./ -# Copy licenses -COPY --from=build-src /usr/share/licenses/testsys /licenses/testsys - -CMD dockerd --storage-driver vfs &>/dev/null & ./metal-k8s-cluster-resource-agent - # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Builds the ECS test agent image FROM public.ecr.aws/amazonlinux/amazonlinux:2 as ecs-test-agent diff --git a/Makefile b/Makefile index 0c446c0e..69ab03b2 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ TESTSYS_BUILD_GOPROXY ?= direct # project AGENT_IMAGES = sonobuoy-test-agent ec2-resource-agent eks-resource-agent ecs-resource-agent \ migration-test-agent vsphere-vm-resource-agent vsphere-k8s-cluster-resource-agent \ - ecs-test-agent k8s-workload-agent ecs-workload-agent metal-k8s-cluster-resource-agent \ + ecs-test-agent k8s-workload-agent ecs-workload-agent \ ec2-karpenter-resource-agent # The set of container images. Add additional artifacts here when added diff --git a/bottlerocket/agents/Cargo.toml b/bottlerocket/agents/Cargo.toml index ac3cd27d..f5b4b3c5 100644 --- a/bottlerocket/agents/Cargo.toml +++ b/bottlerocket/agents/Cargo.toml @@ -28,7 +28,6 @@ k8s-openapi = { version = "0.21", default-features = false, features = ["v1_24"] kube = { version = "0.88", default-features = false, features = ["config", "derive", "client"] } log = "0.4" maplit = "1" -openssh = { version = "0.10", features = ["native-mux"] } testsys-model = { version = "0.0.17", path = "../../model" } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "blocking"] } resource-agent = { version = "0.0.17", path = "../../agent/resource-agent" } diff --git a/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/main.rs b/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/main.rs deleted file mode 100644 index 2d59c857..00000000 --- a/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/main.rs +++ /dev/null @@ -1,46 +0,0 @@ -/*! - -Provides Bottlerocket VMWare vSphere VMs to serve as Kubernetes nodes via `govc`. - -!*/ - -mod metal_k8s_cluster_provider; - -use crate::metal_k8s_cluster_provider::{MetalK8sClusterCreator, MetalK8sClusterDestroyer}; -use agent_utils::init_agent_logger; -use resource_agent::clients::{DefaultAgentClient, DefaultInfoClient}; -use resource_agent::error::AgentResult; -use resource_agent::{Agent, BootstrapData, Types}; -use std::env; -use std::marker::PhantomData; - -#[tokio::main] -async fn main() { - init_agent_logger(env!("CARGO_CRATE_NAME"), None); - let data = match BootstrapData::from_env() { - Ok(ok) => ok, - Err(e) => { - eprintln!("Unable to get bootstrap data: {}", e); - std::process::exit(1); - } - }; - if let Err(e) = run(data).await { - eprintln!("{}", e); - std::process::exit(1); - }; -} - -async fn run(data: BootstrapData) -> AgentResult<()> { - let types = Types { - info_client: PhantomData::, - agent_client: PhantomData::, - }; - let agent = Agent::new( - types, - data, - MetalK8sClusterCreator {}, - MetalK8sClusterDestroyer {}, - ) - .await?; - agent.run().await -} diff --git a/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/metal_k8s_cluster_provider.rs b/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/metal_k8s_cluster_provider.rs deleted file mode 100644 index 2c59717f..00000000 --- a/bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/metal_k8s_cluster_provider.rs +++ /dev/null @@ -1,675 +0,0 @@ -/*! -This is a resource-agent for provisioning an EKS Anywhere Bare Metal k8s cluster. -This agent is used to provision workload clusters. -https://anywhere.eks.amazonaws.com/docs/concepts/clusterworkflow/ -Before using this agent, a management EKS Anywhere cluster needs to be created. Its kubeconfig will -be passed into the agent with `encodedMgmtClusterKubeconfig`. -Once the management cluster has been provisioned, an EKS Anywhere config file needs to be created -for the cluster that needs to be created. -Additionally, a hardware csv corresponding to the cluster config needs to be base64 encoded. - -This agent utilizes workload cluster creation by invoking `eksctl anywhere create cluster` with the -management cluster's kubeconfig, the cluster config, and the hardware csv. -After the cluster has been created, an AWS SSM activation is created for each provisioned node. -The ssh key provided by EKS Anywhere is used to ssh to the admin container and set the ssm activation -with `api client set` allowing ssm docs to be run on the Bare Metal machines. -In addition to the ssm activation, additional userdata can used with `userdata`. -The userdata is accepted in the form of base64 encoded toml and is converted from toml to json and -applied using `apiclient set --json` over ssh via the admin container. - -At deletion, each SSM activation is terminated and the cluster is cleaned up using -`eksctl anywhere delete cluster`. There is a chance that some of the artifacts created by EKS Anywhere -are not cleaned up completely, so `kubectl delete -f` is used on the management cluster to ensure all -k8s artifacts included in the workload cluster config are deleted. -!*/ - -use agent_utils::aws::aws_config; -use agent_utils::base64_decode_write_file; -use agent_utils::ssm::{create_ssm_activation, ensure_ssm_service_role, wait_for_ssm_ready}; -use base64::engine::general_purpose::STANDARD as Base64; -use base64::Engine; -use bottlerocket_agents::clusters::{ - download_eks_a_bundle, install_eks_a_binary, retrieve_workload_cluster_kubeconfig, - write_validate_mgmt_kubeconfig, -}; -use bottlerocket_types::agent_config::{ - CustomUserData, MetalK8sClusterConfig, AWS_CREDENTIALS_SECRET_NAME, -}; -use k8s_openapi::api::core::v1::Node; -use kube::config::Kubeconfig; -use kube::{Api, Config}; -use log::{debug, info}; -use openssh::{KnownHosts, SessionBuilder}; -use resource_agent::clients::InfoClient; -use resource_agent::provider::{ - Create, Destroy, IntoProviderError, ProviderError, ProviderResult, Resources, Spec, -}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::collections::HashSet; -use std::convert::TryFrom; -use std::fmt::Debug; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::time::Duration; -use std::{env, fs}; -use testsys_model::{Configuration, SecretName}; - -const WORKING_DIR: &str = "/local/eksa-work"; - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ProductionMemo { - /// In this resource we put some traces here that describe what our provider is doing. - pub current_status: String, - - pub ssm_activation_id: String, - - /// The name of the secret containing aws credentials. - pub aws_secret_name: Option, - - /// The role that is being assumed. - pub assume_role: Option, - - /// The instance ids for machines provisioned by this agent. - pub instance_ids: HashSet, - - /// The name of the cluster. - pub cluster_name: Option, -} - -impl Configuration for ProductionMemo {} - -/// Once we have fulfilled the `Create` request, we return information about the metal K8s cluster -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub struct CreatedMetalK8sCluster { - /// The base64 encoded kubeconfig for this cluster - pub encoded_kubeconfig: String, - - /// The instance IDs of all SSM-registered machines - pub instance_ids: HashSet, -} - -impl Configuration for CreatedMetalK8sCluster {} - -pub struct MetalK8sClusterCreator {} - -#[async_trait::async_trait] -impl Create for MetalK8sClusterCreator { - type Config = MetalK8sClusterConfig; - type Info = ProductionMemo; - type Resource = CreatedMetalK8sCluster; - - async fn create( - &self, - spec: Spec, - client: &I, - ) -> ProviderResult - where - I: InfoClient, - { - let mut memo: ProductionMemo = client - .get_info() - .await - .context(Resources::Unknown, "Unable to get info from info client")?; - // Keep track of the state of resources - let mut resources = Resources::Clear; - - info!("Initializing agent"); - memo.current_status = "Initializing agent".to_string(); - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending cluster creation message")?; - - memo.aws_secret_name = spec.secrets.get(AWS_CREDENTIALS_SECRET_NAME).cloned(); - memo.assume_role.clone_from(&spec.configuration.assume_role); - - let shared_config = aws_config( - &spec.secrets.get(AWS_CREDENTIALS_SECRET_NAME), - &spec.configuration.assume_role, - &None, - &None, - &None, - false, - ) - .await - .context(Resources::Clear, "Error creating config")?; - - let ssm_client = aws_sdk_ssm::Client::new(&shared_config); - let iam_client = aws_sdk_iam::Client::new(&shared_config); - - info!("Checking SSM service role"); - memo.current_status = "Checking SSM service role".to_string(); - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending cluster creation message")?; - - // Ensure we have a SSM service role we can attach to the VMs - ensure_ssm_service_role(&iam_client) - .await - .context(Resources::Clear, "Unable to check for SSM service role")?; - - info!("Creating working directory"); - memo.current_status = "Creating working directory".to_string(); - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending cluster creation message")?; - - // Set current directory to somewhere other than '/' so eksctl-anywhere won't try to mount - // it in a container. - fs::create_dir_all(WORKING_DIR).context( - resources, - format!("Failed to create working directory '{}'", WORKING_DIR), - )?; - env::set_current_dir(Path::new(WORKING_DIR)).context( - resources, - format!("Failed to change current directory to {}", WORKING_DIR), - )?; - - let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR); - let bundle_manifest_path = format!("{}/bundles.yaml", WORKING_DIR); - let eksa_config_path = format!("{}/cluster.yaml", WORKING_DIR); - let hardware_csv_path = format!("{}/hardware.csv", WORKING_DIR); - base64_decode_write_file(&spec.configuration.hardware_csv_base64, &hardware_csv_path) - .await - .context( - resources, - "Unable to decode and write hardware requirements", - )?; - - let decoded_config = Base64 - .decode(&spec.configuration.cluster_config_base64) - .context(Resources::Clear, "Unable to decode eksctl configuration.")?; - - let deserialized_config = serde_yaml::Deserializer::from_slice(decoded_config.as_slice()) - .map(|config| { - serde_yaml::Value::deserialize(config) - .context(Resources::Clear, "Unable to deserialize eksa config file") - }) - // Make sure all of the configs were deserializable - .collect::>>()?; - - let cluster_name = deserialized_config - .iter() - // Find the `Cluster` config - .find(|config| { - config.get("kind") == Some(&serde_yaml::Value::String("Cluster".to_string())) - }) - // Get the name from the metadata field in the `Cluster` config - .and_then(|config| config.get("metadata")) - .and_then(|config| config.get("name")) - .and_then(|name| name.as_str()) - .context( - Resources::Clear, - "Unable to determine the cluster name from the config file ", - )? - .to_string(); - let cluster_version = deserialized_config - .iter() - // Find the `Cluster` config - .find(|config| { - config.get("kind") == Some(&serde_yaml::Value::String("Cluster".to_string())) - }) - // Get the K8s version from the `spec` field in the `Cluster` config - .and_then(|config| config.get("spec")) - .and_then(|config| config.get("kubernetesVersion")) - .and_then(|ver| ver.as_str()) - .context( - Resources::Clear, - "Unable to determine the cluster K8s version from the config file ", - )? - .to_string(); - - // Set the feature flag for potential new EKS version support in EKS-A - env::set_var( - format!("K8S_{}_SUPPORT", cluster_version.replace('.', "_")), - "true", - ); - - fs::write(&eksa_config_path, decoded_config).context( - Resources::Clear, - format!( - "Unable to write EKS Anywhere configuration to '{}'", - eksa_config_path - ), - )?; - - install_eks_a_binary(&spec.configuration.eks_a_release_manifest_url, &resources).await?; - download_eks_a_bundle( - &spec.configuration.eks_a_release_manifest_url, - &bundle_manifest_path, - &resources, - ) - .await?; - - info!("Creating cluster"); - memo.current_status = "Creating cluster".to_string(); - memo.cluster_name = Some(cluster_name.clone()); - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending cluster creation message")?; - let mgmt_k8s_client = write_validate_mgmt_kubeconfig( - &spec.configuration.mgmt_cluster_kubeconfig_base64, - &mgmt_kubeconfig_path, - &resources, - ) - .await?; - // Call eksctl-anywhere to create cluster with existing mgmt cluster - let status = Command::new("eksctl") - .args(["anywhere", "create", "cluster"]) - .args(["--kubeconfig", &mgmt_kubeconfig_path]) - .args(["--bundles-override", &bundle_manifest_path]) - .args(["-f", &eksa_config_path]) - .args(["--hardware-csv", &hardware_csv_path]) - .arg("--skip-ip-check") - .args(["-v", "4"]) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() - .context(resources, "Failed to launch eksctl process")?; - resources = Resources::Remaining; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!( - "Failed to create EKS-A workload cluster with status code {}", - status - ), - )); - } - let encoded_kubeconfig = - retrieve_workload_cluster_kubeconfig(mgmt_k8s_client, &cluster_name, &resources) - .await?; - - info!("Cluster created"); - - // Now we need the nodes ip addresses to enable SSM. - memo.current_status = "Getting node ips".into(); - - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending final creation message")?; - - let k8s_client = kube::client::Client::try_from( - Config::from_custom_kubeconfig( - Kubeconfig::from_yaml(&String::from_utf8_lossy( - &Base64.decode(&encoded_kubeconfig).context( - resources, - "Unable to decode encoded workload cluster kubeconfig", - )?, - )) - .context( - resources, - "Unable to create workload kubeconfig from encoded config", - )?, - &Default::default(), - ) - .await - .context(resources, "Unable to create `Config` from `Kubeconfig`")?, - ) - .context(resources, "Unable create K8s client from kubeconfig")?; - - let machine_ips = Api::::all(k8s_client) - .list(&Default::default()) - .await - .context(resources, "Unable to list nodes from workload cluster")? - .into_iter() - .map(|node| { - node.status - .and_then(|status| status.addresses) - .unwrap_or_default() - }) - .map(|addresses| { - addresses - .into_iter() - .find(|addr| addr.type_.as_str() == "InternalIP") - .map(|addr| addr.address) - .context(resources, "A node was missing an InternalIp address") - }) - .collect::>>()?; - - // Generate SSM activation codes and IDs - let activation = - create_ssm_activation(&cluster_name, machine_ips.len() as i32, &ssm_client) - .await - .context(resources, "Unable to create SSM activation")?; - activation.0.clone_into(&mut memo.ssm_activation_id); - let control_host_ctr_userdata = json!({"ssm":{"activation-id": activation.0.to_string(), "activation-code":activation.1.to_string(),"region":"us-west-2"}}); - debug!( - "Control container host container userdata: {}", - control_host_ctr_userdata - ); - let ssm_json = json!({ - "host-containers": { - "control": { - "enabled": true, - "user-data": Base64.encode( - control_host_ctr_userdata.to_string()) - } - } - } - ); - - let custom_settings = &spec - .configuration - .custom_user_data - .map(|userdata| match userdata { - CustomUserData::Replace { encoded_userdata } - | CustomUserData::Merge { encoded_userdata } => encoded_userdata, - }) - .map(|userdata| Base64.decode(userdata)) - .transpose() - .context(resources, "Unable to decode custom user data")? - .map(|userdata| toml::from_slice::(&userdata)) - .transpose() - .context(resources, "Unable to deserialize custom userdata")?; - - custom_settings.iter().for_each(|settings| { - info!( - "Custom userdata was deserialized to the following settings:\n{}", - settings.to_string() - ) - }); - - // Enable SSM on the nodes - let mut instance_ids = HashSet::new(); - for machine_ip in machine_ips { - info!("Starting session for {}", machine_ip); - let session = SessionBuilder::default() - .keyfile( - Path::new(WORKING_DIR) - .join(&cluster_name) - .join("eks-a-id_rsa"), - ) - .user("ec2-user".to_string()) - .known_hosts_check(KnownHosts::Accept) - .user_known_hosts_file("/dev/null") - .connect_timeout(Duration::from_secs(5)) - .connect_mux(&machine_ip) - .await - .context( - resources, - format!("Unable to connect to machine with ip '{}'", machine_ip), - )?; - - info!("Getting initial settings for {}", machine_ip); - let status = session - .command("apiclient") - .args(["get", "settings"]) - .status() - .await - .context(resources, "Unable to call `apiclient get settings`")?; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!("Failed to get settings with status code {}", status), - )); - } - - info!("Setting ssm activations for {}", machine_ip); - let status = session - .command("apiclient") - .args(["set", "--json", &ssm_json.to_string()]) - .status() - .await - .context(resources, "Unable to call `apiclient set`")?; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!("Failed to set ssm activation with status code {}", status), - )); - } - - if let Some(settings) = &custom_settings { - info!( - "Settings custom userdata as settings via `apiclient set` for {}", - machine_ip - ); - - let status = session - .command("apiclient") - .args(["set", "--json", &settings.to_string()]) - .status() - .await - .context(resources, "Unable to call `apiclient set`")?; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!("Failed to set custom settings with status code {}", status), - )); - } - } - - let instance_info = tokio::time::timeout( - Duration::from_secs(60), - wait_for_ssm_ready(&ssm_client, &memo.ssm_activation_id, &machine_ip), - ) - .await - .context( - resources, - format!( - "Timed out waiting for SSM agent to be ready on VM '{}'", - machine_ip - ), - )? - .context(resources, "Unable to determine if SSM activation is ready")?; - - instance_ids.insert( - instance_info - .instance_id() - .context( - resources, - format!( - "The instance id was missing for the machine with ip '{}' and activation '{}'", - machine_ip, &memo.ssm_activation_id - ), - )? - .to_string(), - ); - } - - // We are done, set our custom status to say so. - memo.current_status = "Cluster created".into(); - memo.instance_ids.clone_from(&instance_ids); - - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending final creation message")?; - - Ok(CreatedMetalK8sCluster { - instance_ids, - encoded_kubeconfig, - }) - } -} - -pub struct MetalK8sClusterDestroyer {} - -#[async_trait::async_trait] -impl Destroy for MetalK8sClusterDestroyer { - type Config = MetalK8sClusterConfig; - type Info = ProductionMemo; - type Resource = CreatedMetalK8sCluster; - - async fn destroy( - &self, - spec: Option>, - resource: Option, - client: &I, - ) -> ProviderResult<()> - where - I: InfoClient, - { - let mut memo: ProductionMemo = client.get_info().await.map_err(|e| { - ProviderError::new_with_source_and_context( - Resources::Unknown, - "Unable to get info from client", - e, - ) - })?; - let resources = if !memo.instance_ids.is_empty() || !memo.ssm_activation_id.is_empty() { - Resources::Remaining - } else { - Resources::Clear - }; - - let cluster_name = memo.cluster_name.as_ref().context( - resources, - "The cluster name was missing from this agents production memo", - )?; - - let shared_config = aws_config( - &memo.aws_secret_name.as_ref(), - &memo.assume_role, - &None, - &None, - &None, - false, - ) - .await - .context(Resources::Clear, "Error creating config")?; - let ssm_client = aws_sdk_ssm::Client::new(&shared_config); - - // Set current directory to somewhere other than '/' so eksctl-anywhere won't try to mount - // it in a container. - fs::create_dir_all(WORKING_DIR).context( - resources, - format!("Failed to create working directory '{}'", WORKING_DIR), - )?; - env::set_current_dir(Path::new(WORKING_DIR)).context( - resources, - format!("Failed to change current directory to {}", WORKING_DIR), - )?; - - // Set the cluster deletion configs paths - let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR); - let bundle_manifest_path = format!("{}/bundle.yaml", WORKING_DIR); - let eksa_config_path = format!("{}/cluster.yaml", WORKING_DIR); - let workload_kubeconfig_path = - format!("{}/{}-eks-a-cluster.kubeconfig", WORKING_DIR, cluster_name); - - // Populate each file that is needed for cluster deletion - let configuration = spec - .context(resources, "The spec was not provided for destruction")? - .configuration; - - install_eks_a_binary(&configuration.eks_a_release_manifest_url, &resources).await?; - download_eks_a_bundle( - &configuration.eks_a_release_manifest_url, - &bundle_manifest_path, - &resources, - ) - .await?; - - base64_decode_write_file( - &configuration.mgmt_cluster_kubeconfig_base64, - &mgmt_kubeconfig_path, - ) - .await - .context( - resources, - "Unable to decode and write hardware requirements", - )?; - base64_decode_write_file(&configuration.cluster_config_base64, &eksa_config_path) - .await - .context( - resources, - "Unable to decode and write hardware requirements", - )?; - let encoded_kubeconfig = resource - .context( - resources, - "The created resource was not provided for destruction", - )? - .encoded_kubeconfig; - base64_decode_write_file(&encoded_kubeconfig, &workload_kubeconfig_path) - .await - .context( - resources, - "Unable to decode and write hardware requirements", - )?; - - info!("Deleting cluster"); - memo.current_status = "Deleting cluster".to_string(); - client - .send_info(memo.clone()) - .await - .context(resources, "Error sending cluster creation message")?; - // Call eksctl-anywhere to delete cluster with mgmt cluster - let status = Command::new("eksctl") - .args(["anywhere", "delete", "cluster"]) - .args(["--kubeconfig", &mgmt_kubeconfig_path]) - .args(["--bundles-override", &bundle_manifest_path]) - .args(["-f", &eksa_config_path]) - .args(["--w-config", &workload_kubeconfig_path]) - .args(["-v", "4"]) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() - .context(resources, "Failed to launch eksctl process")?; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!( - "Failed to delete EKS-A workload cluster with status code {}", - status - ), - )); - } - - info!("Cleaning up leftover EKSA artifacts."); - // Clean up leftover EKSA Templates - Command::new("kubectl") - .args(["delete"]) - .args(["--kubeconfig", &mgmt_kubeconfig_path]) - .args(["-f", &eksa_config_path]) - .args(["--ignore-not-found"]) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status() - .context(resources, "Failed to launch eksctl process")?; - if !status.success() { - return Err(ProviderError::new_with_context( - resources, - format!( - "Failed to delete EKS-A workload cluster with status code {}", - status - ), - )); - } - - // Deregister managed instances - for instance_id in &memo.instance_ids { - info!("Deregistering {}", instance_id); - ssm_client - .deregister_managed_instance() - .instance_id(instance_id) - .send() - .await - .context( - resources, - format!("Failed deregister managed instance '{}'", instance_id), - )?; - } - - memo.instance_ids.clear(); - info!("Cluster deleted"); - memo.current_status = "Cluster deleted".into(); - client.send_info(memo.clone()).await.map_err(|e| { - ProviderError::new_with_source_and_context( - Resources::Clear, - "Error sending final destruction message", - e, - ) - })?; - - Ok(()) - } -} diff --git a/bottlerocket/samples/Makefile.toml b/bottlerocket/samples/Makefile.toml index 23977ee9..b7a5d3f5 100644 --- a/bottlerocket/samples/Makefile.toml +++ b/bottlerocket/samples/Makefile.toml @@ -46,12 +46,6 @@ SINGLE_IMAGE_REPO = { value = "false", condition = { env_not_set = ["SINGLE_IMAG # GOVC_RESOURCE_POOL = # GOVC_FOLDER = -# The following variables need values only if a metal test file is being created. - -# MGMT_CLUSTER_KUBECONFIG_PATH = -# HARDWARE_CSV_PATH = -# CLUSTER_CONFIG_PATH = - [tasks.install-cli] script=[''' cargo install --path "${CARGO_MAKE_WORKING_DIRECTORY}/../../cli" @@ -79,7 +73,6 @@ SONOBUOY_TEST_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/sonobuoy-test EKS_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/eks-resource-agent:v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["EKS_RESOURCE_AGENT_IMAGE_URI"] } } VSPHERE_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/vsphere-k8s-cluster-resource-agent:v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["VSPHERE_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI"] } } VSPHERE_VM_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/vsphere-vm-resource-agent:v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["VSPHERE_VM_RESOURCE_AGENT_IMAGE_URI"] } } -METAL_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/metal-k8s-cluster-resource-agent:v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["METAL_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI"] } } CONTROLLER_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/controller:v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["CONTROLLER_IMAGE_URI"] } } [tasks.set-agent-images-single-repo] @@ -97,7 +90,6 @@ SONOBUOY_TEST_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE EKS_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE_REPO}:eks-resource-agent-v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["EKS_RESOURCE_AGENT_IMAGE_URI"] } } VSPHERE_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE_REPO}:vsphere-k8s-cluster-resource-agent-v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["VSPHERE_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI"] } } VSPHERE_VM_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE_REPO}:vsphere-vm-resource-agent-v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["VSPHERE_VM_RESOURCE_AGENT_IMAGE_URI"] } } -METAL_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE_REPO}:metal-k8s-cluster-resource-agent-v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["VSPHERE_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI"] } } CONTROLLER_IMAGE_URI = { value = "${AGENT_IMAGE_REGISTRY}/${AGENT_IMAGE_REPO}:controller-v${AGENT_IMAGE_VERSION}", condition = { env_not_set = ["CONTROLLER_IMAGE_URI"] } } [tasks.metadata-url.env] @@ -350,6 +342,3 @@ VARIANT = { value = "vmware-k8s-1.32", condition = { env_not_set = ["VARIANT"] } [tasks.vmware-sonobuoy-test.env] VARIANT = { value = "vmware-k8s-1.32", condition = { env_not_set = ["VARIANT"] } } - -[tasks.metal-sonobuoy-test.env] -VARIANT = { value = "metal-k8s-1.32", condition = { env_not_set = ["VARIANT"] } } diff --git a/bottlerocket/samples/eks/metal-sonobuoy-test.yaml b/bottlerocket/samples/eks/metal-sonobuoy-test.yaml deleted file mode 100644 index 9534e6dd..00000000 --- a/bottlerocket/samples/eks/metal-sonobuoy-test.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: testsys.system/v1 -kind: Test -metadata: - name: ${CLUSTER_NAME}-test - namespace: testsys -spec: - agent: - name: sonobuoy-test-agent - image: ${SONOBUOY_TEST_AGENT_IMAGE_URI} - keepRunning: true - configuration: - assumeRole: ${ASSUME_ROLE} - kubeconfigBase64: \${${CLUSTER_NAME}.encodedKubeconfig} - plugin: "e2e" - mode: ${SONOBUOY_MODE} - dependsOn: [] - resources: [${CLUSTER_NAME}] ---- -apiVersion: testsys.system/v1 -kind: Resource -metadata: - name: ${CLUSTER_NAME} - namespace: testsys -spec: - agent: - name: agent - image: ${METAL_K8S_CLUSTER_RESOURCE_AGENT_IMAGE_URI} - keepRunning: true - privileged: true - configuration: - mgmtClusterKubeconfigBase64: ${MGMT_CLUSTER_KUBECONFIG_BASE64} - hardwareCsvBase64: ${HARDWARE_CSV_BASE64} - assumeRole: ${ASSUME_ROLE} - clusterConfigBase64: ${CLUSTER_CONFIG_BASE64} - dependsOn: [] - destructionPolicy: onDeletion diff --git a/bottlerocket/types/src/agent_config.rs b/bottlerocket/types/src/agent_config.rs index ed685ffb..de151e90 100644 --- a/bottlerocket/types/src/agent_config.rs +++ b/bottlerocket/types/src/agent_config.rs @@ -196,31 +196,6 @@ pub struct VSphereK8sClusterConfig { pub mgmt_cluster_kubeconfig_base64: String, } -/// The configuration information for a eks instance provider. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Default, Configuration, Builder)] -#[serde(rename_all = "camelCase")] -#[crd("Resource")] -pub struct MetalK8sClusterConfig { - /// URL for an EKS-A release manifest that contains URLs for EKS-A binary archives. - /// Defaults to upstream EKS-A release channels. - pub eks_a_release_manifest_url: Option, - - /// Base64-encoded Kubeconfig for the CAPI management cluster - pub mgmt_cluster_kubeconfig_base64: String, - - /// The role that should be assumed when activating SSM for the machines. - pub assume_role: Option, - - /// The base64-encoded EKS Anywhere config for this cluster. - pub cluster_config_base64: String, - - /// The base64-encoded hardware csv that will be used for cluster creation. - pub hardware_csv_base64: String, - - /// Custom TOML data that should be inserted into user-data settings. - pub custom_user_data: Option, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum CreationPolicy { @@ -637,8 +612,8 @@ mod test { use testsys_model::{Resource, Test}; use super::{ - EcsWorkloadTestConfig, EksClusterConfig, MetalK8sClusterConfig, SonobuoyConfig, - VSphereK8sClusterConfig, VSphereVmConfig, WorkloadConfig, + EcsWorkloadTestConfig, EksClusterConfig, SonobuoyConfig, VSphereK8sClusterConfig, + VSphereVmConfig, WorkloadConfig, }; fn samples_dir() -> PathBuf { @@ -1013,31 +988,6 @@ mod test { .unwrap(); } - #[test] - fn metal_sonobuoy_test() { - let s = read_eks_file("metal-sonobuoy-test.yaml"); - let s = s - .replace("${SONOBUOY_MODE}", "quick") - .replace("${", "<") - .replace('}', ">") - .replace('\\', ""); - - let docs: Vec<&str> = s.split("---").collect(); - let &yaml = docs.first().unwrap(); - let test_1_initial: Test = serde_yaml::from_str(yaml).unwrap(); - let _: SonobuoyConfig = serde_json::from_value(JsonValue::Object( - test_1_initial.spec.agent.configuration.unwrap(), - )) - .unwrap(); - - let &yaml = docs.get(1).unwrap(); - let cluster_resource: Resource = serde_yaml::from_str(yaml).unwrap(); - let _: MetalK8sClusterConfig = serde_json::from_value(JsonValue::Object( - cluster_resource.spec.agent.configuration.unwrap(), - )) - .unwrap(); - } - #[test] fn ecs_test_kind() { let s = read_kind_file("ecs-test.yaml");