Discovery configMap is filled properly

Author: Maleware

rust/operator-binary/src/controller.rs

@@ -449,8 +449,8 @@ pub async fn reconcile_hive(
         .context(ApplyRoleBindingSnafu)?;
 
     let mut ss_cond_builder = StatefulSetConditionBuilder::default();
-    let mut listeners = Vec::<Listener>::new();
-    // for each rg a service needs to be build for hive
+    // Collecting listener objects with corresponding rolegroup to fill the discovery configMap later on
+    let mut listener_refs = BTreeMap::<&String, Listener>::new();
     for (rolegroup_name, rolegroup_config) in metastore_config.iter() {
         let rolegroup = hive.metastore_rolegroup_ref(rolegroup_name);
 
@@ -486,7 +486,7 @@ pub async fn reconcile_hive(
             &rolegroup,
             config.listener_class,
         )?;
-        // TODO: Listener name might be wrong
+
         let listener = cluster_resources
             .add(client, rg_group_listener)
             .await
@@ -501,7 +501,7 @@ pub async fn reconcile_hive(
                 rolegroup: rolegroup.clone(),
             })?;
 
-        listeners.push(listener);
+        listener_refs.insert(rolegroup_name, listener);
 
         cluster_resources
             .add(client, rg_configmap)
@@ -534,10 +534,15 @@ pub async fn reconcile_hive(
     // We don't /need/ stability, but it's still nice to avoid spurious changes where possible.
     let mut discovery_hash = FnvHasher::with_key(0);
 
-    for discovery_cm in
-        discovery::build_discovery_configmaps(hive, hive, &resolved_product_image, None, &listeners)
-            .await
-            .context(BuildDiscoveryConfigSnafu)?
+    for discovery_cm in discovery::build_discovery_configmaps(
+        hive,
+        hive,
+        &resolved_product_image,
+        None,
+        listener_refs,
+    )
+    .await
+    .context(BuildDiscoveryConfigSnafu)?
     {
         let discovery_cm = cluster_resources
             .add(client, discovery_cm)

rust/operator-binary/src/discovery.rs

@@ -1,10 +1,10 @@
-use std::num::TryFromIntError;
+use std::{collections::BTreeMap, num::TryFromIntError};
 
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
     builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder},
     commons::product_image_selection::ResolvedProductImage,
-    crd::listener,
+    crd::listener::v1alpha1::Listener,
     k8s_openapi::api::core::v1::{ConfigMap, Service},
     kube::{Resource, runtime::reflector::ObjectRef},
 };
@@ -56,6 +56,8 @@ pub enum Error {
     MetadataBuild {
         source: stackable_operator::builder::meta::Error,
     },
+    #[snafu(display("failed to apply group listener for {rolegroup}"))]
+    RoleGroupListenerHasNoAddress { rolegroup: String },
 }
 
 /// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::HiveCluster`] for all expected
@@ -65,7 +67,7 @@ pub async fn build_discovery_configmaps(
     hive: &v1alpha1::HiveCluster,
     resolved_product_image: &ResolvedProductImage,
     chroot: Option<&str>,
-    listeners: &[listener::v1alpha1::Listener],
+    listener_refs: BTreeMap<&String, Listener>,
 ) -> Result<Vec<ConfigMap>, Error> {
     let name = owner
         .meta()
@@ -79,7 +81,7 @@ pub async fn build_discovery_configmaps(
         hive,
         resolved_product_image,
         chroot,
-        listener_hosts(listeners, HIVE_PORT_NAME)?,
+        listener_refs,
     )?];
 
     Ok(discovery_configmaps)
@@ -95,67 +97,56 @@ fn build_discovery_configmap(
     hive: &v1alpha1::HiveCluster,
     resolved_product_image: &ResolvedProductImage,
     chroot: Option<&str>,
-    hosts: impl IntoIterator<Item = (impl Into<String>, u16)>,
+    listener_refs: BTreeMap<&String, Listener>,
 ) -> Result<ConfigMap, Error> {
-    let mut conn_str = hosts
-        .into_iter()
-        .map(|(host, port)| format!("thrift://{}:{}", host.into(), port))
-        .collect::<Vec<_>>()
-        .join("\n");
-    if let Some(chroot) = chroot {
-        if !chroot.starts_with('/') {
-            return RelativeChrootSnafu { chroot }.fail();
+    let mut discovery_configmap = ConfigMapBuilder::new();
+
+    discovery_configmap.metadata(
+        ObjectMetaBuilder::new()
+            .name_and_namespace(hive)
+            .name(name)
+            .ownerreference_from_resource(owner, None, Some(true))
+            .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu {
+                hive: ObjectRef::from_obj(hive),
+            })?
+            .with_recommended_labels(build_recommended_labels(
+                hive,
+                &resolved_product_image.app_version_label,
+                &HiveRole::MetaStore.to_string(),
+                "discovery",
+            ))
+            .context(MetadataBuildSnafu)?
+            .build(),
+    );
+
+    for (rolegroup, listener_ref) in listener_refs {
+        let listener_address = listener_ref
+            .status
+            .and_then(|s| s.ingress_addresses?.into_iter().next())
+            .context(RoleGroupListenerHasNoAddressSnafu { rolegroup })?;
+        let mut conn_str = format!(
+            "thrift://{}:{}",
+            listener_address.address,
+            listener_address
+                .ports
+                .get(HIVE_PORT_NAME)
+                .copied()
+                .context(NoServicePortSnafu {
+                    port_name: HIVE_PORT_NAME
+                })?
+        );
+        if let Some(chroot) = chroot {
+            if !chroot.starts_with('/') {
+                return RelativeChrootSnafu { chroot }.fail();
+            }
+            conn_str.push_str(chroot);
         }
-        conn_str.push_str(chroot);
+        discovery_configmap.add_data(rolegroup, conn_str);
     }
-    ConfigMapBuilder::new()
-        .metadata(
-            ObjectMetaBuilder::new()
-                .name_and_namespace(hive)
-                .name(name)
-                .ownerreference_from_resource(owner, None, Some(true))
-                .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu {
-                    hive: ObjectRef::from_obj(hive),
-                })?
-                .with_recommended_labels(build_recommended_labels(
-                    hive,
-                    &resolved_product_image.app_version_label,
-                    &HiveRole::MetaStore.to_string(),
-                    "discovery",
-                ))
-                .context(MetadataBuildSnafu)?
-                .build(),
-        )
-        .add_data("HIVE", conn_str)
+
+    discovery_configmap
         .build()
         .with_context(|_| DiscoveryConfigMapSnafu {
             obj_ref: ObjectRef::from_obj(hive),
         })
 }
-
-fn listener_hosts(
-    listeners: &[listener::v1alpha1::Listener],
-    port_name: &str,
-) -> Result<impl IntoIterator<Item = (String, u16)>, Error> {
-    listeners
-        .iter()
-        .flat_map(|listener| {
-            listener
-                .status
-                .as_ref()
-                .and_then(|s| s.ingress_addresses.as_deref())
-        })
-        .flatten()
-        .map(|addr| {
-            Ok((
-                addr.address.clone(),
-                addr.ports
-                    .get(port_name)
-                    .copied()
-                    .context(NoServicePortSnafu { port_name })?
-                    .try_into()
-                    .context(InvalidNodePortSnafu)?,
-            ))
-        })
-        .collect::<Result<Vec<_>, _>>()
-}

tests/ansible/roles/expand-tests/CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [Unreleased]
+
+### Changed
+
+- Generated kuttl tests prepend the original test name to the generated kuttl test folders ([#1])
+
+[#1]: https://github.com/stackabletech/expand-tests/pull/1
+

tests/ansible/roles/expand-tests/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

tests/ansible/roles/expand-tests/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+work_dir: "../test-work"
+test_dir: "../tests"
+template_dir: "{{ test_dir }}/templates"

tests/ansible/roles/expand-tests/files/generate_tests.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import argparse
+from argparse import Namespace
+from dataclasses import dataclass, field, asdict
+import itertools
+import logging
+import sys
+from typing import Dict
+import yaml
+from itertools import chain
+
+
+@dataclass
+class TestCase:
+    testcase: str
+    values: Dict[str, str]
+    name: str = field(init=False)
+
+    def __post_init__(self):
+        self.name = "_".join(
+            chain(
+                [self.testcase],
+                ["-".join([x, self.values.get(x)]) for x in self.values.keys()],
+            )
+        )
+
+
+def check_args() -> Namespace:
+    parser = argparse.ArgumentParser(
+        description="This tool is used by the Stackable integration tests to create the final matrix of tests to run "
+        "based on test dimensions defined for the test suite it is running in."
+    )
+    parser.add_argument(
+        "--input",
+        "-i",
+        required=True,
+        help="The input file (yaml format) which specifies the dimensions and their values",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        required=True,
+        help="A yaml file to be read by Ansible which contains the test cases",
+    )
+    parser.add_argument(
+        "--debug",
+        "-d",
+        action="store_true",
+        required=False,
+        help="Will print additional debug statements (e.g. output from all run commands)",
+    )
+    args = parser.parse_args()
+
+    log_level = "DEBUG" if args.debug else "INFO"
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s %(levelname)s: %(message)s",
+        stream=sys.stdout,
+    )
+    return args
+
+
+def main() -> int:
+    args = check_args()
+    result = []
+    with open(args.input, encoding="utf8") as stream:
+        input_dimensions = yaml.safe_load(stream)
+
+        for test_case in input_dimensions["tests"]:
+            dimensions = test_case["dimensions"]
+            used_dimensions = [
+                v for v in input_dimensions["dimensions"] if v["name"] in dimensions
+            ]
+            tmp = []
+            for value in used_dimensions:
+                tmp.append([(value["name"], x) for x in value["values"]])
+
+            for materialized_case in itertools.product(*tmp):
+                result.append(
+                    TestCase(testcase=test_case["name"], values=dict(materialized_case))
+                )
+
+        with open(args.output, "w", encoding="utf8") as outstream:
+            outputstruct = {"tests": [asdict(r) for r in result]}
+            logging.debug(f"Got the following output: {outputstruct}")
+            yaml.dump(outputstruct, outstream)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/ansible/roles/expand-tests/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for expand-tests

tests/ansible/roles/expand-tests/meta/.galaxy_install_info

@@ -0,0 +1,2 @@
+install_date: Wed Jul 13 08:18:32 2022
+version: ''