dev-setup/docker-compose-kafka-connect.yml at master · mmahmoodictbd/dev-setup · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
version: '3.3'

services:

  kafka-zookeeper:
    container_name: kafka-zookeeper
    hostname: kafka-zookeeper
    image: wurstmeister/zookeeper
    ports:
      - 2181:2181

  kafka:
    container_name: kafka
    hostname: kafka
    image: wurstmeister/kafka
    command: [start-kafka.sh]
    ports:
      - 9092:9092
      - 29092:29092
    environment:
      KAFKA_ADVERTISED_HOST_NAME: localhost # docker-machine ip
      KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://localhost:9092"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: kafka-zookeeper:2181
      KAFKA_ADVERTISED_PORT: 9092
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - kafka-zookeeper

  kafdrop:
    container_name: kafdrop
    image: obsidiandynamics/kafdrop
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: "kafka:29092"
      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
    depends_on:
      - kafka

  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:latest
    hostname: kafka-schema-registry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: kafka-zookeeper:2181
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081

  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:latest
    container_name: kafka-rest-proxy
    hostname: kafka-rest-proxy
    ports:
      - "8082:8082"
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: kafka-zookeeper:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: kafka1:9092
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/

  kafka-connect:
    image: confluentinc/cp-kafka-connect:latest
    container_name: kafka-connect
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: kafka:9092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: compose-connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
      CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://kafka-schema-registry:8081'
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: '/usr/share/confluent-hub-components/'
    command:
      # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
      - bash
      - -c
      - |
        echo "Installing connector plugins"
        confluent-hub install --no-prompt confluentinc/kafka-connect-influxdb:latest
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run &
        #
        echo "Waiting for Kafka Connect to start listening on $$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT ⏳"
        while : ; do
          curl_status=$$(curl -s -o /dev/null -w %{http_code} http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors)
          echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)"
          if [ $$curl_status -eq 200 ] ; then
            break
          fi
          sleep 5
        done

        # while : ; do
        #  curl_status=$$(curl -s -o /dev/null -w %{http_code} --request POST http://$$CONNECT_REST_ADVERTISED_HOST_NAME:$$CONNECT_REST_PORT/connectors --header 'Content-Type: application/json' --raw '{ "name": "InfluxSinkConnector", "config":{ "connector.class": "io.confluent.influxdb.InfluxDBSinkConnector", "topics": "VirtualTemperatureSensor", "tasks.max": "1", "measurement.name.format" : "${topic}", "influxdb.url": "http://influxdb:8086", "influxdb.db": "db0", "influxdb.kcql": "INSERT INTO VirtualTemperatureSensor SELECT * FROM VirtualTemperatureSensor" } }')
        # echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)"
        # if [ $$curl_status -eq 200 ] ; then
        #    break
        #  fi
        #  sleep 5
        # done
        # echo "Launching Kafka Connect worker"
        # /etc/confluent/docker/run &
        #
        # echo "Waiting for Schema Registry to start listening on schema-registry:8081 ⏳"
        # while [ $$(curl -s -o /dev/null -w %{http_code} http://schema-registry:8081) -eq 000 ] ; do
        #  echo -e $$(date) " Schema Registry listener HTTP state: " $$(curl -s -o /dev/null -w %{http_code} http://schema-registry:8081) " (waiting for != 000)"
        #  sleep 5
        # done
        #
        sleep infinity
    depends_on:
      - kafka-zookeeper
      - kafka
      - kafka-rest-proxy

#  YET ANOTHER APPROTCH
#  fast-data-dev:
#    image: lensesio/fast-data-dev:latest
#    container_name: fast-data-dev
#    ports:
#      - "3030:3030" # Admin interface
#    depends_on:
#      - influxdb
#    environment:
#      - ADV_HOST=fast-data-dev
#      - FORWARDLOGS=0
#      - RUNTESTS=0
#      - CONNECTORS=influxdb
#      - DEBUG=1