Heka Kafka Producer Output

1. Sample Configuration

filename               = "kafka.lua"
message_matcher        = "TRUE"
output_limit           = 8 * 1024 * 1024
brokerlist             = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205
ticker_interval        = 60
async_buffer_size      = 20000
ignore_topic_err       = false
ignore_partition_err   = false

topic_constant = "test"
producer_conf = {
    ["queue.buffering.max.messages"] = async_buffer_size,
    ["batch.num.messages"] = 200,
    ["message.max.bytes"] = output_limit,
    ["queue.buffering.max.ms"] = 10,
    ["topic.metadata.refresh.interval.ms"] = -1,
}

-- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties
topic_confs = {
    -- <topic_name> = {<topic_conf>},
    ["*"] = {<topic_conf>} -- optional default topic configuration
}

-- Specify a module that will encode/convert the Heka message into its output representation.
encoder_module = "encoders.heka.protobuf" -- default

source code: kafka.lua