How to send messages to same topic using kafka producer ?

Question

I have setup kafka client that produce and consume messages, it is working as expected when we send payloads from producer to topic, So i have problem with producer now first message i was able to send it to topic and i was also able to consume it from kafka topic , Now i am tryig to send second messages but consumer is not reading the second message from kafka topic, Any idea what is happening here ?

producer.js

var config = require('./config.js');
var zk = require('node-zookeeper-client');
var kafkaConn = "130.8";
var kafka = require('kafka-node'),
    HighLevelProducer = kafka.HighLevelProducer,
    client = new kafka.Client(kafkaConn),
    producer = new HighLevelProducer(client),
    payloads = [
        { topic: 'test', messages: 'second message' }
    ];
producer.on('ready', function () {
    producer.send(payloads, function (err, data) {
        console.log(data);
    });
});

consumer.js

function start () {
    topics = [{topic: 'test'}];
    var groupId = 'ulogGroup';
    var clientId = "consumer-" + Math.floor(Math.random() * 10000);
    var options = {autoCommit: true, fetchMaxWaitMs: 100, fetchMaxBytes: 10 * 1024 * 1024, groupId: groupId};
    console.log("Started consumer: ", clientId);
    var consumer_client = new kafka.Client(kafkaConn,clientId);
    var client = new Client(consumer_client.connectionString,clientId);
    var consumer = new HighLevelConsumer(client, topics, options);
    console.log("Consumer topics:", getConsumerTopics(consumer).toString());
   // startConsumer(consumer);
    consumer.on('message', function (message) {
        //var topic = message.data;
        console.log('Message',message);
    });
};
start();

Mukul Mukul · Accepted Answer · 2016-07-07T07:06:40

Could be the most trivial suggestion, but are you sure your second message is being sent to kafka queue ? Alongside your consumer, you may want to use inbuilt command line consumer while your producer is on work to publish messages, just to make sure all your messages are being published.

bin/kafka-console-consumer.sh --zookeeper host:2181 --topic test --from-beginning

How to send messages to same topic using kafka producer ?

1 Answers