GCP BigQuery Loader

Batches message data into new line delimited JSON or TSV files and loads them into BigQuery when they reach the specified size or timeout.

0.0.1. Sample Configuration

filename                = "gcp_bigquery.lua"
message_matcher         = "Type == 'bigquery.json'"
ticker_interval         = 60
read_queue              = "input"
shutdown_on_terminate   = true

-- directory location to store the intermediate output files
batch_dir       = "/var/tmp" -- default

-- Specifies how much data (in bytes) can be written to a single file before
-- it is finalized.
max_file_size       = 1024 * 1024 * 1024 -- default

-- Specifies how long (in seconds) to wait before the file is finalized
-- Idle files are only checked every ticker_interval seconds.
max_file_age        = 3600 -- default
load_fail_cache     = 50 -- default number of files to save off for manual recovery
bq_dataset          = "test"
bq_table            = "demo"
--json_field        = "Payload" -- default should contain a single line of JSON with no new lines
--tsv_fields        = {"Timestamp", "Type", "Payload", "Fields[error_detail]", "Fields[data]"}

-- Specify a module that will encode/convert the Heka message into its output representation.
encoder_module = nil -- uses the standard behavior documented above

alert = {
  disabled = false,
  prefix = true,
  throttle = 0,
  modules = {
    email = {recipients = {"notify@example.com"}},
  },
}

source code: gcp_bigquery.lua