Примеры работы с kafka

Publishing Messages to a Topic in Kafka


$ kafka-topics.sh \
--create \
--zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka \
--replication-factor 1 \
--partitions 3 \
--topic test


$ kafka-topics.sh --zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka --topic test --describe
Topic:test	PartitionCount:3	ReplicationFactor:1	Configs:
Topic: test	Partition: 0	Leader: 1	Replicas: 1	Isr: 1
Topic: test	Partition: 1	Leader: 2	Replicas: 2	Isr: 2
Topic: test	Partition: 2	Leader: 1	Replicas: 1	Isr: 1


// Отправить сообщения
$ kafka-console-producer.sh \
--broker-list kafka1:9092,kafka2:9092,kafka3:9092 \
--topic test
>message1
>message2
>message3
>^C



$ kafka-topics.sh --zookeeper zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/kafka --topic test --describe

// Получить все сообщения
$ kafka-console-consumer.sh --bootstrap-server kafka1:9092,kafka2:9092,kafka3:9092 --topic test --from-beginning

// Получить только новые сообщения
$ kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic test


Brokers

$ zookeeper-shell.sh kafka1:2181

ls /kafka
[cluster, controller_epoch, controller, brokers, admin, isr_change_notification, consumers, log_dir_event_notification, latest_producer_id_block, config


get /controller
{"version":1,"brokerid":3,"timestamp":"1583181928919"}
cZxid = 0x100000044
ctime = Mon Mar 02 23:45:28 MSK 2020
mZxid = 0x100000044
mtime = Mon Mar 02 23:45:28 MSK 2020
pZxid = 0x100000044
cversion = 0
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x1709cfef7c10001
dataLength = 54
numChildren = 0