Add kafka 3 node cluster

Neenad Ingole
1 parent a423f995
Showing 3 changed files with 170 additions and 0 deletions Show diff stats
images/multi-broker-kafka-cluster.png
kafka-cluster/README.md
kafka-cluster/docker-compose.yml
@@ -0,0 +1,33 @@
+## Apache Kafka cluster
+
+This docker compose provides 3 broker nodes and 1 zookeeper node Kafka Cluster.
+This helps to test Kafka application on local instead of a real cluster.
+
+The configurations are bare minimum to start the server and could be changed based on requirement
+
+There is also a Kafka UI application available at http://localhost:8080
+
+![](../images/multi-broker-kafka-cluster.png)
+
+### How to start
+    
+```bash
+docker-compose up
+
+use -d to run in detached mode
+```
+> If you don't see changes to the cluster after updating the yml, try doing
+> `docker compose down` and then start the cluster again.
+
+### Port Mapping
+| Port | Description             |
+|------|-------------------------|
+| 2181 | Zookeeper               |
+| 9092 | Kafka Broker 1          |
+| 9093 | Kafka Broker 2          |
+| 9094 | Kafka Broker 3          |
+| 8080 | Kafka UI                |
+| 1099 | JMX PORT on all brokers |
+
+
+
@@ -0,0 +1,137 @@
+version: '3'
+
+services:
+  zookeeper-kafka:
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - '2181:2181'
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    healthcheck:
+      # fake healthcheck just to have one to simplify status check
+      test: ['CMD', 'date']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  # docs at: https://registry.hub.docker.com/r/wurstmeister/kafka
+  kafka-broker-1:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9092:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://:9092'
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-1 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_CREATE_TOPICS: 'test:3:3'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-1:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-broker-2:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9093:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 2
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://:9092'
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-2 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-2:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  kafka-broker-3:
+    image: wurstmeister/kafka:latest
+    restart: always
+    depends_on:
+      - zookeeper-kafka
+    ports:
+      - '9094:9092'
+    volumes:
+      # It's the only way to deal with Kafka non-static exposed ports to host
+      # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      KAFKA_BROKER_ID: 3
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
+      PORT_COMMAND: 'docker port $$(hostname) 9092/tcp | cut -d: -f2'
+      KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://:9092'
+      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-3 -Dcom.sun.management.jmxremote.rmi.port=1099"
+      JMX_PORT: 1099
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka/bin/kafka-topics.sh',
+          '--list',
+          '--bootstrap-server',
+          'kafka-broker-3:29092',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-ui:
+    container_name: kafka-ui
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - "8080:8080"
+    depends_on:
+      - zookeeper-kafka
+      - kafka-broker-1
+      - kafka-broker-2
+      - kafka-broker-3
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-broker-1:29092, kafka-broker-2:29092, kafka-broker-3:29092
+      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-kafka:2181
+      KAFKA_CLUSTERS_0_JMXPORT: 1099
@@ -0,0 +1,33 @@		@@ -0,0 +1,33 @@
	1	+## Apache Kafka cluster
	2	+
	3	+This docker compose provides 3 broker nodes and 1 zookeeper node Kafka Cluster.
	4	+This helps to test Kafka application on local instead of a real cluster.
	5	+
	6	+The configurations are bare minimum to start the server and could be changed based on requirement
	7	+
	8	+There is also a Kafka UI application available at http://localhost:8080
	9	+
	10	+![](../images/multi-broker-kafka-cluster.png)
	11	+
	12	+### How to start
	13	+
	14	+```bash
	15	+docker-compose up
	16	+
	17	+use -d to run in detached mode
	18	+```
	19	+> If you don't see changes to the cluster after updating the yml, try doing
	20	+> `docker compose down` and then start the cluster again.
	21	+
	22	+### Port Mapping
	23	+\| Port \| Description \|
	24	+\|------\|-------------------------\|
	25	+\| 2181 \| Zookeeper \|
	26	+\| 9092 \| Kafka Broker 1 \|
	27	+\| 9093 \| Kafka Broker 2 \|
	28	+\| 9094 \| Kafka Broker 3 \|
	29	+\| 8080 \| Kafka UI \|
	30	+\| 1099 \| JMX PORT on all brokers \|
	31	+
	32	+
	33	+
@@ -0,0 +1,137 @@		@@ -0,0 +1,137 @@
	1	+version: '3'
	2	+
	3	+services:
	4	+ zookeeper-kafka:
	5	+ image: confluentinc/cp-zookeeper:latest
	6	+ ports:
	7	+ - '2181:2181'
	8	+ environment:
	9	+ ZOOKEEPER_CLIENT_PORT: 2181
	10	+ ZOOKEEPER_TICK_TIME: 2000
	11	+ healthcheck:
	12	+ # fake healthcheck just to have one to simplify status check
	13	+ test: ['CMD', 'date']
	14	+ interval: 10s
	15	+ timeout: 5s
	16	+ retries: 5
	17	+ # docs at: https://registry.hub.docker.com/r/wurstmeister/kafka
	18	+ kafka-broker-1:
	19	+ image: wurstmeister/kafka:latest
	20	+ restart: always
	21	+ depends_on:
	22	+ - zookeeper-kafka
	23	+ ports:
	24	+ - '9092:9092'
	25	+ volumes:
	26	+ # It's the only way to deal with Kafka non-static exposed ports to host
	27	+ # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
	28	+ - /var/run/docker.sock:/var/run/docker.sock
	29	+ environment:
	30	+ KAFKA_BROKER_ID: 1
	31	+ KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
	32	+ PORT_COMMAND: 'docker port $$(hostname) 9092/tcp \| cut -d: -f2'
	33	+ KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://:9092'
	34	+ KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-1:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
	35	+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
	36	+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
	37	+ KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-1 -Dcom.sun.management.jmxremote.rmi.port=1099"
	38	+ JMX_PORT: 1099
	39	+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
	40	+ KAFKA_CREATE_TOPICS: 'test:3:3'
	41	+ healthcheck:
	42	+ test:
	43	+ [
	44	+ 'CMD',
	45	+ '/opt/kafka/bin/kafka-topics.sh',
	46	+ '--list',
	47	+ '--bootstrap-server',
	48	+ 'kafka-broker-1:29092',
	49	+ ]
	50	+ interval: 10s
	51	+ timeout: 5s
	52	+ retries: 5
	53	+
	54	+ kafka-broker-2:
	55	+ image: wurstmeister/kafka:latest
	56	+ restart: always
	57	+ depends_on:
	58	+ - zookeeper-kafka
	59	+ ports:
	60	+ - '9093:9092'
	61	+ volumes:
	62	+ # It's the only way to deal with Kafka non-static exposed ports to host
	63	+ # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
	64	+ - /var/run/docker.sock:/var/run/docker.sock
	65	+ environment:
	66	+ KAFKA_BROKER_ID: 2
	67	+ KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
	68	+ PORT_COMMAND: 'docker port $$(hostname) 9092/tcp \| cut -d: -f2'
	69	+ KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://:9092'
	70	+ KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-2 -Dcom.sun.management.jmxremote.rmi.port=1099"
	71	+ JMX_PORT: 1099
	72	+ KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-2:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
	73	+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
	74	+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
	75	+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
	76	+ healthcheck:
	77	+ test:
	78	+ [
	79	+ 'CMD',
	80	+ '/opt/kafka/bin/kafka-topics.sh',
	81	+ '--list',
	82	+ '--bootstrap-server',
	83	+ 'kafka-broker-2:29092',
	84	+ ]
	85	+ interval: 10s
	86	+ timeout: 5s
	87	+ retries: 5
	88	+ kafka-broker-3:
	89	+ image: wurstmeister/kafka:latest
	90	+ restart: always
	91	+ depends_on:
	92	+ - zookeeper-kafka
	93	+ ports:
	94	+ - '9094:9092'
	95	+ volumes:
	96	+ # It's the only way to deal with Kafka non-static exposed ports to host
	97	+ # See: https://github.com/wurstmeister/kafka-docker/blob/master/start-kafka.sh#L65-L76
	98	+ - /var/run/docker.sock:/var/run/docker.sock
	99	+ environment:
	100	+ KAFKA_BROKER_ID: 3
	101	+ KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-kafka:2181'
	102	+ PORT_COMMAND: 'docker port $$(hostname) 9092/tcp \| cut -d: -f2'
	103	+ KAFKA_LISTENERS: 'LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://:9092'
	104	+ KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=kafka-broker-3 -Dcom.sun.management.jmxremote.rmi.port=1099"
	105	+ JMX_PORT: 1099
	106	+ KAFKA_ADVERTISED_LISTENERS: LISTENER_INTERNAL://kafka-broker-3:29092,LISTENER_HOST://localhost:_{PORT_COMMAND}
	107	+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_INTERNAL:PLAINTEXT,LISTENER_HOST:PLAINTEXT
	108	+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_INTERNAL
	109	+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
	110	+ healthcheck:
	111	+ test:
	112	+ [
	113	+ 'CMD',
	114	+ '/opt/kafka/bin/kafka-topics.sh',
	115	+ '--list',
	116	+ '--bootstrap-server',
	117	+ 'kafka-broker-3:29092',
	118	+ ]
	119	+ interval: 10s
	120	+ timeout: 5s
	121	+ retries: 5
	122	+
	123	+ kafka-ui:
	124	+ container_name: kafka-ui
	125	+ image: provectuslabs/kafka-ui:latest
	126	+ ports:
	127	+ - "8080:8080"
	128	+ depends_on:
	129	+ - zookeeper-kafka
	130	+ - kafka-broker-1
	131	+ - kafka-broker-2
	132	+ - kafka-broker-3
	133	+ environment:
	134	+ KAFKA_CLUSTERS_0_NAME: local
	135	+ KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-broker-1:29092, kafka-broker-2:29092, kafka-broker-3:29092
	136	+ KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper-kafka:2181
	137	+ KAFKA_CLUSTERS_0_JMXPORT: 1099