Support KRaft for consensus building (#889)

* extract common properties to role config

* rename KafkaConfig to BrokerConfig

* rename Container to BrokerContainer

* make zookeeper config map ref optional

* remove zookeeper.connect and timeout from product config

* add controller role and move configs in own module

* fix tests

* regenerated charts

* use broker.properties and controller.properties

* replace missing server.properties

* remove unused errors

* add controller to product confif machinery

* remove cp

* add controller to security.properties

* add controller to security properties

* wip - add AnyConfig to access different role configs

* remove errors from product logging

* finished controller prep work

* remove clones

* remove obsolete methods

* cleanup

* use role methods

* remove obsolete errors

* remove unwrap

* wip - kafka controller cleanup, move resources to own module

* wip - controller working

* wip - combine broker & controller

* fix todos

* zookeeper working again

* fix todos

* kraftify smoke test

* kraftify tests definition

* revert test changes

* wip - controller tls

* fix 3.7.x commands

* hash node.id offset from rolegroup

* started kerberos tests

* adapt logging tests

* error out on node id hash collision

* fix cluster operation tests

* fix configuration tests

* fix upgrade tests

* start fix remove rolegroup tests

* started kraft docs

* fix unit tests

* fix docs

* run pre commit

* fix yaml lint

* clarify docs, clean up

* linter

* improve jaas config

* add comments, remove todo

* remove redundant memory increase

* implement node.id hashing check for all roles

* fix linter

* prepare  log4j2 config

* consolidate cluster operation tests

* consolidate delete rolegroup

* adjust kerberos tests

* fix logging tests

* fix smoke test

* fix tls tests

* fix upgrade tests

* Apply suggestions from code review

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

* do not apply znodes in kraftmode

* use cluster name instead of uid for kafka cluster-id

* replace uid with name for cluster-id

* add pre-stop sleep hook to controller kafka container

* add "kraft" test suite

* add experimental warning to kraft docs

* add kafka 4.0.0 and log4j2

* Apply suggestions from code review

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

* remove explicit filenames from doc paragraph

* add quotes to clarify code docs

* add "smoke-kraft" test

* add "operations-kraft" test

* update operations test

* tests: remove zookeeper=false tests

* tests: remove zookeeper-latest: false

* add 4.1.0

* added known issues section

* tests: missed one

* improve known issues

* remove log4j vs log4j2 java arg todo

* fix / improve todos

* fix copy paste

* tests: fix tls tests

* tests: disable scaling for kraft 3.7

* remove  obsolete zk/kraft checks

* add gracefulShutdownTimeout to stabilize kerberos test

---------

Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>

Author: maltesander

.vscode/launch.json

@@ -6,13 +6,17 @@
             "request": "launch",
             "name": "Debug operator binary",
             "cargo": {
-                "args": ["build"],
+                "args": [
+                    "build"
+                ],
                 "filter": {
                     "name": "stackable-{[ operator.name }]",
                     "kind": "bin"
                 }
             },
-            "args": ["run"],
+            "args": [
+                "run"
+            ],
             "cwd": "${workspaceFolder}"
         }
     ]

deploy/config-spec/properties.yaml

@@ -1,3 +1,4 @@
+---
 version: 0.1.0
 spec:
   units:
@@ -35,6 +36,8 @@ properties:
       roles:
         - name: "broker"
           required: true
+        - name: "controller"
+          required: true
       asOfVersion: "0.0.0"
       comment: "TTL for successfully resolved domain names."
       description: "TTL for successfully resolved domain names."
@@ -54,52 +57,18 @@ properties:
       roles:
         - name: "broker"
           required: true
+        - name: "controller"
+          required: true
       asOfVersion: "0.0.0"
       comment: "TTL for domain names that cannot be resolved."
       description: "TTL for domain names that cannot be resolved."
 
-  - property: &zookeeperConnect
-      propertyNames:
-        - name: "zookeeper.connect"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "string"
-        unit: *unitUrl
-      defaultValues:
-        - fromVersion: "0.0.0"
-          value: "localhost:2181"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "The zookeeper connection string"
-
-  - property: &zookeeperTimeout
-      propertyNames:
-        - name: "zookeeper.connection.timeout.ms"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "integer"
-        unit: *unitMilliseconds
-      defaultValues:
-        - fromVersion: "0.0.0"
-          value: "18000"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "Zookeeper connection timeout in milliseconds."
-
   - property: &opaAuthorizerClassName
       propertyNames:
         - name: "authorizer.class.name"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "string"
       defaultValues:
@@ -118,7 +87,7 @@ properties:
         - name: "opa.authorizer.url"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "string"
         unit: *unitUrl
@@ -133,7 +102,7 @@ properties:
         - name: "opa.authorizer.cache.initial.capacity"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -151,7 +120,7 @@ properties:
         - name: "opa.authorizer.cache.maximum.size"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -169,7 +138,7 @@ properties:
         - name: "opa.authorizer.cache.expire.after.seconds"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -181,19 +150,3 @@ properties:
           required: false
       asOfVersion: "0.0.0"
       description: "The number of seconds after which the OPA authorizer cache expires"
-
-  - property: &logDirs
-      propertyNames:
-        - name: "log.dirs"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "string"
-      recommendedValues:
-        - value: "/stackable/data/topicdata"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "A comma separated list of directories under which to store log files"

deploy/helm/kafka-operator/configs/properties.yaml

@@ -1,3 +1,4 @@
+---
 version: 0.1.0
 spec:
   units:
@@ -35,6 +36,8 @@ properties:
       roles:
         - name: "broker"
           required: true
+        - name: "controller"
+          required: true
       asOfVersion: "0.0.0"
       comment: "TTL for successfully resolved domain names."
       description: "TTL for successfully resolved domain names."
@@ -54,52 +57,18 @@ properties:
       roles:
         - name: "broker"
           required: true
+        - name: "controller"
+          required: true
       asOfVersion: "0.0.0"
       comment: "TTL for domain names that cannot be resolved."
       description: "TTL for domain names that cannot be resolved."
 
-  - property: &zookeeperConnect
-      propertyNames:
-        - name: "zookeeper.connect"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "string"
-        unit: *unitUrl
-      defaultValues:
-        - fromVersion: "0.0.0"
-          value: "localhost:2181"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "The zookeeper connection string"
-
-  - property: &zookeeperTimeout
-      propertyNames:
-        - name: "zookeeper.connection.timeout.ms"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "integer"
-        unit: *unitMilliseconds
-      defaultValues:
-        - fromVersion: "0.0.0"
-          value: "18000"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "Zookeeper connection timeout in milliseconds."
-
   - property: &opaAuthorizerClassName
       propertyNames:
         - name: "authorizer.class.name"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "string"
       defaultValues:
@@ -118,7 +87,7 @@ properties:
         - name: "opa.authorizer.url"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "string"
         unit: *unitUrl
@@ -133,7 +102,7 @@ properties:
         - name: "opa.authorizer.cache.initial.capacity"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -151,7 +120,7 @@ properties:
         - name: "opa.authorizer.cache.maximum.size"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -169,7 +138,7 @@ properties:
         - name: "opa.authorizer.cache.expire.after.seconds"
           kind:
             type: "file"
-            file: "server.properties"
+            file: "broker.properties"
       datatype:
         type: "integer"
         unit: *unitCapacity
@@ -181,19 +150,3 @@ properties:
           required: false
       asOfVersion: "0.0.0"
       description: "The number of seconds after which the OPA authorizer cache expires"
-
-  - property: &logDirs
-      propertyNames:
-        - name: "log.dirs"
-          kind:
-            type: "file"
-            file: "server.properties"
-      datatype:
-        type: "string"
-      recommendedValues:
-        - value: "/stackable/data/topicdata"
-      roles:
-        - name: "broker"
-          required: true
-      asOfVersion: "0.0.0"
-      description: "A comma separated list of directories under which to store log files"