Heka Protobuf Message S3 Output Partitioner

Batches message data into Heka protobuf stream files based on the specified path dimensions and copies them to S3 when they reach the maximum size or maximum age.

0.0.1. Sample Configuration

filename        = "moz_telemetry_s3.lua"
message_matcher = "Type == 'telemetry'"
ticker_interval = 60

-- see the moz_telemetry.s3 module
dimension_file  = "test.json"

-- https://bugzilla.mozilla.org/show_bug.cgi?id=1353110
experiment_dimension_file  = "test_experiments.json" -- optional

-- directory location to store the intermediate standard output files
batch_dir       = "/var/tmp/foobar"

-- directory location to store the intermediate experiment output files
experiment_batch_dir       = "/var/tmp/experiment"

-- Specifies how many data files to keep open at once. If there are more
-- "current" files than this, the least-recently used file will be closed
-- and then re-opened if more messages arrive before it is copied to S3. The
-- default is 1000. A value of 0 means no maximum.
max_file_handles    = 1000

-- Specifies how much data (in bytes) can be written to a single file before
-- it is finalized (default 300MiB)
max_file_size       = 1024 * 1024 * 300

-- Specifies how long (in seconds) to wait before the file is finalized
-- (default 1 hour).  Idle files are only checked every ticker_interval seconds.
max_file_age        = 60 * 60

-- Specifies that all local files will finalized before exiting (default false).
-- When streaming compression is used the file will always be finalized on exit.
flush_on_shutdown   = true
preserve_data       = not flush_on_shutdown -- should always be the inverse of flush_on_shutdown

-- Specify an optional module to encode incoming messages via its encode function.
-- encoder_module = "encoders.heka.framed_protobuf" -- default

-- Specifies experiment types whitelist, and experiment id blocklist
experiment_types = {["normandy-preference-"] = true} -- optional
experiment_blocklist = {["pref-flip-screenshots-release-1369150"] = true} -- optional

source code: moz_telemetry_s3.lua