Skip to content

Commit 2c6a9b4

Browse files
JohnPrestonJohnPreston
committed
Resources for Confluent Apps blog post
1 parent bb94063 commit 2c6a9b4

11 files changed

Lines changed: 561 additions & 0 deletions

File tree

‎README.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
============================
2+
ComposeX Blog resources
3+
============================
4+
5+
This repository stores the resources for you to use when following our `blog <blog.ecs-composex.lambda-my-aws.io>`.

‎confluent-apps-01/README.rst‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
====================
2+
Confluent Apps
3+
====================
4+
5+
connect-cluster
6+
===============
7+
8+
The connect cluster that we deploy in our AWS environment to deploy a connect cluster.
9+
10+
11+
control-center
12+
==============
13+
14+
The control center allows to have a GUI which allows developers and administrators to interact with the kafka and connect
15+
clusters.
16+
The control center is deployed behind a load-balancer (ALB) which allows remote access. There is only 1 container running and
17+
we do not need more than that.
18+
19+
20+
Secrets management
21+
==================
22+
23+
The secrets management is handled in the exact same way as for any other applications, however, the schema changes based
24+
on what the applications need. Use the templates accordingly.
25+
26+
27+
Deployment
28+
===========
29+
30+
AWS ECS
31+
--------
32+
33+
All services are deployed in AWS ECS using AWS Fargate for compute provisioning.
34+
In nonprod environments, they are on a 2:1 ratio between FARGATE and FARGATE_SPOT.
35+
36+
Deploying using composex
37+
-------------------------
38+
39+
40+
.. code-block:: bash
41+
42+
python -m venv venv
43+
source venv/bin/activate
44+
pip install pip -U
45+
pip install ecs_composex>=0.8.9
46+
# AWS_PROFILE is the name of the profile you have authed in using SSO.
47+
# When running from codebuild or else, this does not need to be specified.
48+
# ENV_NAME can be one of dev, stg or prod.
49+
ENV_NAME=dev ecs-composex up -n kafka--confluent-apps-${ENV_NAME} -f docker-compose.yml -envs/${ENV_NAME}.yml
50+
51+
52+
.. warning::
53+
54+
There can only be one Control Center running presently per Kafka cluster, or you need to assign random IDs to these.

‎confluent-apps-01/connect-cluster/Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
FROM confluentinc/cp-kafka-connect:5.5.1
2+
RUN apt-get update && apt-get install jq -y && rm -rf /var/lib/apt/lists/*
3+
COPY start.sh /etc/confluent/docker/start.sh
4+
CMD ["/etc/confluent/docker/start.sh"]

‎confluent-apps-01/connect-cluster/connect-cluster_secrets.yaml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
AWSTemplateFormatVersion: "2010-09-09"
2+
Description: |
3+
Template to create a new AWS Secret storing credentials for a consumer group of a specific confluent kafka cluster
4+
5+
Metadata:
6+
AWS::CloudFormation::Interface:
7+
ParameterGroups:
8+
- Label:
9+
default: Cluster Settings
10+
Parameters:
11+
- ClusterId
12+
- BootstrapEndpoint
13+
- BootstrapPort
14+
- SchemaRegistryUrl
15+
- ClusterName
16+
- Label:
17+
default: Connect general settings
18+
Parameters:
19+
- ClusterConnectUsername
20+
- ClusterConnectPassword
21+
- Label:
22+
default: Connect consumer settings
23+
Parameters:
24+
- ClusterConsumerUsername
25+
- ClusterConsumerPassword
26+
- Label:
27+
default: Connect producer settings
28+
Parameters:
29+
- ClusterProducerUsername
30+
- ClusterProducerPassword
31+
- Label:
32+
default: Schema Registry credentials
33+
Parameters:
34+
- SchemaRegistryGroupUsername
35+
- SchemaRegistryGroupPassword
36+
37+
Parameters:
38+
ClusterId:
39+
Type: String
40+
BootstrapEndpoint:
41+
Type: String
42+
BootstrapPort:
43+
Type: Number
44+
MinValue: 1024
45+
MaxValue: 65535
46+
47+
ClusterName:
48+
Type: String
49+
50+
ClusterConnectUsername:
51+
Type: String
52+
NoEcho: True
53+
ClusterConnectPassword:
54+
Type: String
55+
NoEcho: True
56+
57+
ClusterProducerUsername:
58+
Type: String
59+
NoEcho: True
60+
ClusterProducerPassword:
61+
Type: String
62+
NoEcho: True
63+
64+
ClusterConsumerUsername:
65+
Type: String
66+
NoEcho: True
67+
ClusterConsumerPassword:
68+
Type: String
69+
NoEcho: True
70+
71+
72+
SchemaRegistryUrl:
73+
Type: String
74+
SchemaRegistryGroupUsername:
75+
Type: String
76+
NoEcho: True
77+
SchemaRegistryGroupPassword:
78+
Type: String
79+
NoEcho: True
80+
81+
Resources:
82+
KafkaSecret:
83+
Type: 'AWS::SecretsManager::Secret'
84+
Properties:
85+
Name: !Sub '/kafka/${ClusterId}/${ClusterName}'
86+
Description: "Secret for Confluent Connect Cluster credentials"
87+
SecretString: !Sub |
88+
{
89+
"SCHEMA_REGISTRY_URL": "${SchemaRegistryUrl}",
90+
"SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
91+
"CONNECT_BOOTSTRAP_SERVERS": "${BootstrapEndpoint}:${BootstrapPort}",
92+
"CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL": "${SchemaRegistryUrl}",
93+
"CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
94+
"CONNECT_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ClusterConnectUsername}\" password=\"${ClusterConnectPassword}\";",
95+
"CONNECT_CONSUMER_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ClusterConsumerUsername}\" password=\"${ClusterConsumerPassword}\";",
96+
"CONNECT_PRODUCER_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ClusterProducerUsername}\" password=\"${ClusterProducerPassword}\";",
97+
"CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ClusterConsumerUsername}\" password=\"${ClusterConsumerPassword}\";",
98+
"CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ClusterProducerUsername}\" password=\"${ClusterProducerPassword}\";"
99+
}

‎confluent-apps-01/connect-cluster/start.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
3+
export CONNECT_REST_ADVERTISED_HOST_NAME=$(hostname)
4+
echo $CONNECT_REST_ADVERTISED_HOST_NAME
5+
6+
IFS=$'\n'
7+
for s in $(echo $CONNECT_CREDS | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do export $s ; done
8+
/etc/confluent/docker/run || exit 1

‎confluent-apps-01/control-center/Dockerfile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM confluentinc/cp-enterprise-control-center:5.5.1
2+
RUN apt-get update && apt-get install jq -y && rm -rf /var/lib/apt/lists/*
3+
4+
COPY start.sh /etc/confluent/docker/start.sh
5+
CMD ["/etc/confluent/docker/start.sh"]

‎confluent-apps-01/control-center/control_center_secrets.yaml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
AWSTemplateFormatVersion: "2010-09-09"
2+
Description: |
3+
Template to create a new AWS Secret storing credentials for a consumer group of a specific confluent kafka cluster
4+
If you
5+
6+
Metadata:
7+
Author: johnpreston
8+
AWS::CloudFormation::Interface:
9+
ParameterGroups:
10+
- Label:
11+
default: Cluster Settings
12+
Parameters:
13+
- ClusterId
14+
- BootstrapEndpoint
15+
- BootstrapPort
16+
- SchemaRegistryUrl
17+
- Label:
18+
default: Consumer group settings
19+
Parameters:
20+
- ConsumerGroupName
21+
- ConsumerGroupUsername
22+
- ConsumerGroupPassword
23+
- Label:
24+
default: Schema Registry credentials
25+
Parameters:
26+
- SchemaRegistryGroupUsername
27+
- SchemaRegistryGroupPassword
28+
29+
Parameters:
30+
ClusterId:
31+
Type: String
32+
BootstrapEndpoint:
33+
Type: String
34+
BootstrapPort:
35+
Type: Number
36+
MinValue: 1024
37+
MaxValue: 65535
38+
39+
ConsumerGroupName:
40+
Type: String
41+
ConsumerGroupUsername:
42+
Type: String
43+
NoEcho: True
44+
ConsumerGroupPassword:
45+
Type: String
46+
NoEcho: True
47+
48+
SchemaRegistryUrl:
49+
Type: String
50+
SchemaRegistryGroupUsername:
51+
Type: String
52+
NoEcho: True
53+
SchemaRegistryGroupPassword:
54+
Type: String
55+
NoEcho: True
56+
License:
57+
Default: none
58+
Type: String
59+
NoEcho: True
60+
61+
Conditions:
62+
NoLicense: !Equals [ !Ref License, "none" ]
63+
64+
Resources:
65+
KafkaSecret:
66+
Type: 'AWS::SecretsManager::Secret'
67+
Properties:
68+
Name: !Sub '/kafka/${ClusterId}/${ConsumerGroupName}'
69+
Description: "Secret for Confluent Control Center"
70+
SecretString: !If
71+
- NoLicense
72+
- !Sub |
73+
{
74+
"SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
75+
"CONTROL_CENTER_BOOTSTRAP_SERVERS": "${BootstrapEndpoint}:${BootstrapPort}",
76+
"CONTROL_CENTER_SCHEMA_REGISTRY_URL": "${SchemaRegistryUrl}",
77+
"CONTROL_CENTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
78+
"CONTROL_CENTER_STREAMS_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ConsumerGroupUsername}\" password=\"${ConsumerGroupPassword}\";",
79+
"CONFLUENT_LICENSE": "${License}",
80+
"CONTROL_CENTER_LICENSE": "${License}"
81+
}
82+
- !Sub |
83+
{
84+
"SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
85+
"CONTROL_CENTER_BOOTSTRAP_SERVERS": "${BootstrapEndpoint}:${BootstrapPort}",
86+
"CONTROL_CENTER_SCHEMA_REGISTRY_URL": "${SchemaRegistryUrl}",
87+
"CONTROL_CENTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO": "${SchemaRegistryGroupUsername}:${SchemaRegistryGroupPassword}",
88+
"CONTROL_CENTER_STREAMS_SASL_JAAS_CONFIG": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"${ConsumerGroupUsername}\" password=\"${ConsumerGroupPassword}\";",
89+
}

‎confluent-apps-01/control-center/start.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
for c in $CONNECT_CLUSTERS; do
4+
NAME=$(echo $c | awk -F\:\: '{print $1}')
5+
URL=$(echo $c | awk -F\:\: '{print $2}')
6+
echo $NAME - $URL
7+
export CONFLUENT_CONTROL_CENTER_CONNECT_${NAME}_CLUSTER=$URL
8+
export CONTROL_CENTER_CONNECT_${NAME}_CLUSTER=$URL
9+
env | grep CONTROL_CENTER_CONNECT
10+
done
11+
12+
CONFLUENT_CONTROLCENTER_ID=$RANDOM
13+
CONTROL_CENTER_ID=$RANDOM
14+
15+
IFS=$'\n'
16+
for s in $(echo $CC_CREDS | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]" ); do
17+
export $s
18+
done
19+
echo "STARTING CONTROL CENTER"
20+
/etc/confluent/docker/run

‎confluent-apps-01/docker-compose.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
---
2+
version: '3.8'
3+
services:
4+
controlcenter:
5+
build:
6+
context: control-center
7+
dockerfile: Dockerfile
8+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/confluentinc/cp-enterprise-control-center:5.5.1
9+
deploy:
10+
resources:
11+
reservations:
12+
cpus: "1.0"
13+
memory: "2G"
14+
ports:
15+
- 8080:8080
16+
environment:
17+
CONTROL_CENTER_SCHEMA_REGISTRY_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO
18+
CONTROL_CENTER_STREAMS_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
19+
CONTROL_CENTER_STREAMS_SECURITY_PROTOCOL: SASL_SSL
20+
CONTROL_CENTER_STREAMS_SASL_MECHANISM: PLAIN
21+
CONTROL_CENTER_REPLICATION_FACTOR: 3
22+
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_REPLICATION: 3
23+
CONTROL_CENTER_INTERNAL_TOPICS_REPLICATION: 3
24+
CONTROL_CENTER_COMMAND_TOPIC_REPLICATION: 3
25+
CONTROL_CENTER_METRICS_TOPIC_REPLICATION: 3
26+
CONFLUENT_METRICS_TOPIC_REPLICATION: 3
27+
CONTROL_CENTER_STREAMS_NUM_STREAM_THREADS: 3
28+
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
29+
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
30+
# Workaround for MMA-3564
31+
CONTROL_CENTER_METRICS_TOPIC_MAX_MESSAGE_BYTES: 8388608
32+
CONTROL_CENTER_REST_LISTENERS: "http://0.0.0.0:8080"
33+
PORT: 8080
34+
secrets:
35+
- CC_CREDS
36+
depends_on:
37+
- connect
38+
39+
connect:
40+
build:
41+
context: connect-cluster
42+
dockerfile: Dockerfile
43+
image: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/confluentinc/cp-kafka-connect:5.5.1
44+
ports:
45+
- 8083:8083
46+
environment:
47+
CONNECT_REST_PORT: 8083
48+
CONNECT_GROUP_ID: "connect"
49+
CONNECT_CONFIG_STORAGE_TOPIC: "connect-configs"
50+
CONNECT_OFFSET_STORAGE_TOPIC: "connect-offsets"
51+
CONNECT_STATUS_STORAGE_TOPIC: "connect-status"
52+
CONNECT_REPLICATION_FACTOR: 3
53+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3
54+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3
55+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3
56+
57+
CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
58+
CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
59+
60+
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
61+
CONNECT_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO
62+
63+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
64+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
65+
66+
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
67+
CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
68+
CONNECT_LOG4J_LOGGERS: org.reflections=ERROR
69+
70+
# CLASSPATH required due to CC-2422
71+
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.5.1.jar
72+
73+
# Connect worker
74+
CONNECT_SECURITY_PROTOCOL: SASL_SSL
75+
CONNECT_SASL_MECHANISM: PLAIN
76+
CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
77+
# Connect producer
78+
CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_SSL
79+
CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
80+
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
81+
CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL
82+
CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN
83+
# Connect consumer
84+
CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_SSL
85+
CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
86+
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
87+
CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL
88+
CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN
89+
TZ: "Europe/London"
90+
secrets:
91+
- CONNECT_CREDS

0 commit comments

Comments
 (0)