# drain.conf

#############
## MODULES ##
#############

module(
  load="impstats"
  interval="<%= @data["log_drain"]["stats_interval_seconds"] %>"
  format="json"
  ruleset="rs_stats"
)

module(
  load="imhttp"
  ports="<%= @data["log_drain"]["port"] %>"
  liboptions=[
    "access_log_file=<%= @data["log_drain"]["http_access_log_file"] %>",
    "error_log_file=<%= @data["log_drain"]["http_error_log_file"] %>",
    "num_threads=<%= @data["log_drain"]["http_num_threads"] %>",
    "listen_backlog=<%= @data["log_drain"]["http_listen_backlog"] %>",
    "connection_queue=<%= @data["log_drain"]["http_connection_queue"] %>"
  ]
  <%- if @data["log_drain"]["debug"] -%>
  documentroot="<%= @data["log_drain"]["http_document_root"] %>"
  <%- end -%>
)

module(load="mmfields")
module(load="mmjsonparse")
module(load="mmnormalize")
module(load="omprog")

###################
## LOOKUP TABLES ##
###################

# Heroku metadata lookup tables.
lookup_table(name="heroku_apps" file="/etc/rsyslog.d/tables/heroku-apps.json")
lookup_table(name="heroku_spaces" file="/etc/rsyslog.d/tables/heroku-spaces.json")
lookup_table(name="heroku_regions" file="/etc/rsyslog.d/tables/heroku-regions.json")
lookup_table(name="heroku_addons" file="/etc/rsyslog.d/tables/heroku-addons.json")


##############
## COUNTERS ##
##############

# General counters.
dyn_stats(name="heroku" resettable="off")

# Number of events this drain has seen counted by type.
dyn_stats(name="drain.events.total" resettable="off")

# Number of app events counted per source and type.
dyn_stats(name="app.events.total" resettable="off")

# Total of router request events.
dyn_stats(name="router.requests.total" resettable="off")

# Total of custom events
dyn_stats(name="custom.events.total" resettable="off")

############
## INPUTS ##
############

input(
  type="imhttp"
  name="drain"
  endpoint="/v1/logs/<%= @data["log_drain"]["uuid"]%>"
  <%- if @data["log_drain"]["basic_auth_file"] -%>
  basicAuthFile="<%= @data["log_drain"]["basic_auth_file"]%>"
  <%- end -%>
  ruleset="rs_drain"
  addmetadata="on"
  SupportOctetCountedFraming="on"
  <%- if @data["log_drain"]["disable_flow_control"] -%>
  flowcontrol="off"
  <%- end -%>
)

###############
## TEMPLATES ##
###############

# Event timestamp.
template(name="tpl_timestamp_rfc3339" type="list") {
  property(name="timestamp" dateFormat="rfc3339")
}

# App event counter tag template.
# Format: "<app-id>|<source>|<dyno>|<type>|<app>|<space>|<region>|<team>|<service>".
# e.g. "bbd9f128-380d-4de9-af4b-4d62b44a8dfd|app|web.1|log".
template(name="tpl_heroku_app_event_tags" type="list") {
  property(name="$.app_id") constant(value="|") # App ID or Drain Token.
  property(name="$.source") constant(value="|") # Source.
  property(name="$.dyno")   constant(value="|") # Dyno.
  property(name="$.type")   constant(value="|") # Event type.
  property(name="$.app")    constant(value="|") # App
  property(name="$.space")  constant(value="|") # Space
  property(name="$.region") constant(value="|") # Region
  property(name="$.team")   constant(value="|") # Team
  property(name="$.service") # Service
}

# Router request counter tag template.
# Format: "<app-id>|<dyno>|<method>|<host>|<status>|<heroku-code>|<app>|<space>|<region>|<team>|<service>|<path>".
# e.g. "bbd9f128-380d-4de9-af4b-4d62b44a8dfd|web.1|POST|log-drain.herokuapp.com|200|-".
# e.g. "bbd9f128-380d-4de9-af4b-4d62b44a8dfd|-|POST|log-drain.herokuapp.com|503|H10".
template(name="tpl_heroku_router_request_tags" type="list") {
  property(name="$.app_id")   constant(value="|") # App ID or Drain Token.
  property(name="$.dyno")     constant(value="|") # Dyno.
  property(name="$.method")   constant(value="|") # HTTP method
  property(name="$.host")     constant(value="|") # HTTP host.
  property(name="$.status")   constant(value="|") # HTTP status.
  property(name="$.code")     constant(value="|") # Heroku error code.
  property(name="$.app")      constant(value="|") # App
  property(name="$.space")    constant(value="|") # Space
  property(name="$.region")   constant(value="|") # Region
  property(name="$.team")     constant(value="|") # Team
  property(name="$.service")  constant(value="|") # Service
  property(name="$.path")     # HTTP path that was called.
}

# template for impstats_to_json plugin
template(name="tpl_impstats_json_record" type="list") {
  property(name="$!stats")
  constant(value="\n")
}

# Custom event counter tag template
# Format: "<app-id>|<dyno>|<type>|<context>|<status>|<app>|<space>|<region>|<team>|<service>".
template(name="tpl_custom_event_tags" type="list") {
  property(name="$.app_id")   constant(value="|") # App ID or Drain Token.
  property(name="$.dyno")     constant(value="|") # Dyno.
  property(name="$.type")     constant(value="|") # Event type
  property(name="$.context")  constant(value="|") # Event context
  property(name="$.status")   constant(value="|") # Event status
  property(name="$.app")      constant(value="|") # App
  property(name="$.space")    constant(value="|") # Space
  property(name="$.region")   constant(value="|") # Region
  property(name="$.team")     constant(value="|") # Team
  property(name="$.service") # Service
}

##########################
## RULESET COUNT EVENTS ##
##########################

# Increment event counters. Caller should set the required variables.
# - $.type
# - $.source
# - $.dyno
# - $.app_id
ruleset(name="rs_count_events") {
  set $.inc = dyn_inc("drain.events.total", $.type);
  set $.inc = dyn_inc("app.events.total", exec_template("tpl_heroku_app_event_tags"));
}

###################
## RULESET DRAIN ##
###################

# Tee events to the correct ruleset based on the event type.
ruleset(
  name="rs_drain"
  queue.size="<%= @data["log_drain"]["input_queue_size"] %>"
  <%- if @data["log_drain"]["logs_input_batch_size"] -%>
  queue.dequeueBatchSize="<%= @data["log_drain"]["logs_input_batch_size"] %>"
  <%- end -%>
  <%- if @data["log_drain"]["queue_filename"] -%>
  queue.filename="<%= @data["log_drain"]["queue_filename"]%>"
  queue.spoolDirectory="<%= @data["log_drain"]["queue_spool_directory"]%>"
  queue.maxFileSize="<%= @data["log_drain"]["queue_max_file_size"]%>"
  <%- if @data["log_drain"]["queue_high_watermark"] -%>
  queue.highWatermark="<%= @data["log_drain"]["queue_high_watermark"]%>"
  <%- end -%>
  <%- if @data["log_drain"]["queue_low_watermark"] -%>
  queue.lowWatermark="<%= @data["log_drain"]["queue_low_watermark"]%>"
  <%- end -%>
  <%- end -%>
) {
  unset $!metadata!httpheaders!authorization;
  unset $!metadata!queryparams!mirror;

  # Guard against badly parsed events so we don't produce garbage values.
  if ($app-name != ["app", "heroku", "token"]) then {
    set $.inc = dyn_inc("heroku", "invalid.events.total");
    set $.app_id = "-";
    set $.source = "-";
    set $.dyno = "-";
  } else {
    # If HTTP Header "Logplex-Drain-Token" exists, then logs are coming from Logplex.
    # In this case, use the Drain Token as the unique App ID.
    # Otherwise, we're looking at logs from a Space level drain, where the App ID is the syslog hostname.
    if (strlen($!metadata!httpheaders!logplex-drain-token) > 0) then {
      set $.app_id = $!metadata!httpheaders!logplex-drain-token;
    } else {
      set $.app_id = $hostname;
    }
    set $.source = $app-name;
    set $.dyno = $procid;
  }

  # set metadata info for msg
  set $.app    = lookup("heroku_apps", $.app_id);
  set $.space  = lookup("heroku_spaces", $.app_id);
  set $.region = lookup("heroku_regions", $.app_id);
  set $.team = "-";
  set $.service = "-";
  set $.sourcetype = "-";
  set $.environment = "<%= @data["log_drain"]["environment"] %>";
  set $.cloud = "<%= @data["log_drain"]["cloud"] %>";
  set $.business_unit = "<%= @data["log_drain"]["business_unit"] %>";

  # override with query param if they exist, fix-up encoded space characters
  if ($!metadata!queryparams!app != "") then {
    reset $.app = replace($!metadata!queryparams!app, "%20", " ");
  }
  if ($!metadata!queryparams!space != "") then {
    reset $.space = replace($!metadata!queryparams!space, "%20", " ");
  }
  if ($!metadata!queryparams!region != "") then {
    reset $.region = replace($!metadata!queryparams!region, "%20", " ");
  }
  if ($!metadata!queryparams!service != "") then {
    reset $.service = replace($!metadata!queryparams!service, "%20", " ");
  }
  if ($!metadata!queryparams!team != "") then {
    reset $.team = replace($!metadata!queryparams!team, "%20", " ");
  }
  if ($!metadata!queryparams!sourcetype != "") then {
    reset $.sourcetype = replace($!metadata!queryparams!sourcetype, "%20", " ");
  }
  if ($!metadata!queryparams!cloud != "") then {
    reset $.cloud = replace($!metadata!queryparams!cloud, "%20", " ");
  }
  if ($!metadata!queryparams!environment != "") then {
    reset $.environment = replace($!metadata!queryparams!environment, "%20", " ");
  }
  if ($!metadata!queryparams!bu != "") then {
    reset $.business_unit = replace($!metadata!queryparams!bu, "%20", " ");
  }

  <%- if @data["log_drain"]["runtime_metrics"] -%>
  if ($msg startswith "dyno=" or $msg startswith "source=") then {
    call rs_runtime_metrics
  }
  <%- end -%>
  call rs_logs
  if ($msg startswith "event=") then {
    call rs_custom_metrics
  }
  <%- if @data["log_drain"]["router_metrics"] -%>
  if ($.dyno == "router") then {
    call rs_router_metrics
  }
  <%- end -%>
  <%- if @data["log_drain"]["debug"] -%>
  call rs_debug_files
  <%- end -%>
}


<%- if @data["log_drain"]["debug"] -%>
#########################
## RULESET DEBUG FILES ##
#########################

ruleset(name="rs_debug_files") {
  set $.dynadir = "original";
  action(
    type="omfile"
    dynafile="tpl_dynafile"
  )
  set $.dynadir = "debug";
  action(
    type="omfile"
    template="RSYSLOG_DebugFormat"
    dynafile="tpl_dynafile"
  )
}
<%- end -%>


############################
## RULESET ROUTER METRICS ##
############################

# Parse Heroku router logs as metrics and increment internal counters.
ruleset(name="rs_router_metrics" queue.size="1000") {
  # Possible fields from various router logs.
  #   at=info
  #   method=GET
  #   path="/"
  #   host=app.herokuapp.com
  #   request_id=<uuid>
  #   fwd="<ip>"
  #   dyno=web.1
  #   connect=0ms
  #   service=3ms
  #   status=304
  #   bytes=0
  #   protocol=http
  #   tls_version=tls1.3
  #   code=H10
  #   desc="App crashed"
  action(type="mmfields" separator=" " jsonRoot=".fields")
  foreach ($.field in $.fields) do {
    if ($.field!value startswith "method=") then {
      set $.method = field($.field!value, 61, 2);
    } else if ($.field!value startswith "host=") then {
      set $.host = field($.field!value, 61, 2);
    } else if ($.field!value startswith "dyno=") then {
      set $.dyno = field($.field!value, 61, 2);
    } else if ($.field!value startswith "status=") then {
      set $.status = field($.field!value, 61, 2);
    } else if ($.field!value startswith "code=") then {
      set $.code = field($.field!value, 61, 2);
    } else if ($.field!value startswith "path=") then {
      set $.path = field($.field!value, 61, 2);
    }

  }

  # The dyno field is empty under some errors conditions, default it.
  if (strlen($.dyno) == 0) then {
    set $.dyno = "-";
  }
  # The code field is empty if the event is not an error, default it.
  if (strlen($.code) == 0) then {
    set $.code = "-";
  }

  set $.request_tags = exec_template("tpl_heroku_router_request_tags");
  set $.inc = dyn_inc("router.requests.total", $.request_tags);
  <%- if @data["log_drain"]["debug"] -%>
  set $.dynadir = "debug-router-events";
  action(
    type="omfile"
    template="RSYSLOG_DebugFormat"
    dynafile="tpl_dynafile"
  )
  <%- end -%>
}

#############################
## RULESET RUNTIME METRICS ##
#############################

# Parse and publish Heroku runtime-metrics.
ruleset(name="rs_runtime_metrics") {
  # Parse a few different possible metric sources using mmnormalize.
  # Addon:
  #   source=<attachment> addon=<addon> <rest> ...
  #   source=KAFKA addon=kafka-opaque-17963 sample#load-avg-1m=0 sample#load-avg-5m=0 ...
  # Dyno:
  #   dyno=<blob> source=<dyno> <rest> ... Note that the dyno here is *not* the actual Dyno name, rather a few ID's.
  #   source=<blob> dyno=<dyno> <rest> ... Or reversed. Match with prefix in mmnormalize.
  #   dyno=heroku.c273506e-9ddc-47b6-a40f-b15f6b94c828.c8855b1b-c6d9-4304-865e-5d379989948c source=web.1 sample#memory_total=265.59MB ...
  #   source=web.1 dyno=heroku.c273506e-9ddc-47b6-a40f-b15f6b94c828.c8855b1b-c6d9-4304-865e-5d379989948c sample#memory_total=265.59MB ...
  action(
    type="mmnormalize"
    path="$.runtime"
    rule=[
      "prefix=source=%source:word% ",
      "rule=:addon=%addon:word% %rest:rest%",
      "rule=:dyno=%dyno:word% %rest:rest%",
      "prefix=",
      "rule=:dyno=%dyno:word% source=%source:word% %rest:rest%"
    ]
  )

  if (strlen($.runtime!addon) > 0) then {
    # If the addons table is populated, try to find the associated app.
    set $.app_id = lookup("heroku_addons", $.runtime!addon);
    if ($.app_id == "-") then {
      set $.app_id = $hostname;
    }
    # Additional tags for addon metrics.
    # The addon field reflects the addon instance name.
    # The source field reflects the attachment instance name.
    set $!metric_tags!addon = $.runtime!addon;
    set $!metric_tags!attachment = $.runtime!source;

    # Addons postgres, redis, and kafka get their own metric heirarchy.
    # Postgres and redis can be identified by their dyno name exactly.
    # Kafka dyno names are "heroku-kafka.<n>".
    if ($.dyno == ["heroku-postgres", "heroku-redis"]) then {
      set $!metric_service = $.dyno;
    } else if ($.dyno startswith "heroku-kafka") then {
      set $!metric_service = "heroku-kafka";
    } else {
      # Catch-all for unknown addons or formats.
      set $!metric_service = "heroku-addon";
    }
  } else {
    # Extract the app id from the dyno field in the runtime-metric, e.g. "dyno=heroku.<app-id>.<dyno-id>".
    # We have the app id in the $hostname variable from private space apps, but not for common runtime apps.
    set $.app_id = field($.runtime!dyno, 46, 2); # 2nd field by ascii 46 "."
    set $!metric_service = "heroku-runtime";
  }

  # now that the addon's app_id is set, try to retrieve any missed metadata
  if ($.app == "-") then {
    reset $.app = lookup("heroku_apps", $.app_id);
  }
  if ($.space == "-") then {
    reset $.space = lookup("heroku_spaces", $.app_id);
  }
  if ($.region == "-") then {
    reset $.region = lookup("heroku_regions", $.app_id);
  }

  # Count once we've resolved the app id.
  set $.type = "metric";
  call rs_count_events

  # Timestamp.
  set $!metric_timestamp = parse_time($timestamp);

  # Tags.
  set $!metric_tags!source = $.source;
  set $!metric_tags!dyno   = $.dyno;
  set $!metric_tags!device = $.app_id;
  set $!metric_tags!app    = $.app;
  set $!metric_tags!space  = $.space;
  set $!metric_tags!region = $.region;
  set $!metric_tags!uuid   = "<%= @data["log_drain"]["uuid"] %>";
  set $!metric_tags!team = $.team;
  set $!metric_tags!service = $.service;

  # Parse the message into space-separated fields using mmfields.
  # { "f1": "source=KAFKA", "f2": "addon=kafka-opaque-17963", "f3": "sample#load-avg-1m=0", ... }
  action(type="mmfields" separator=" " jsonRoot=".fields")

  foreach ($.field in $.fields) do {
    if ($.field!value startswith "sample#") then {
      set $.sample  = field($.field!value, 35, 2);  # 2nd field by ascii 35 "#"
      set $.sample_key   = field($.sample, 61, 1);  # 1st field by ascii 61 "="
      set $.sample_value = field($.sample, 61, 2);  # 2nd field by ascii 61 "="
      # Regex parses integers, floats, and units.
      # 0.53591 -> (0.53591)
      # 0 -> (0)
      # 10528428kB -> (10528428, kB)
      <%- re = "([[:digit:]]+(\\\\.[[:digit:]]+)?)([[:alpha:]]+)?" -%>
      set $.value = re_extract($.sample_value, "<%= re %>", 0, 1, "0"); # Full match (group 0) subgroup 1
      set $.units = re_extract($.sample_value, "<%= re %>", 0, 3, "");  # Full match (group 0) subgroup 3

      set $!metric_value = $.value;
      if (strlen($.units) > 0) then {
        set $!metric_name = $.sample_key & "." & $.units;
      } else {
        set $!metric_name = $.sample_key;
      }

      <%- if @data["log_drain"]["debug"] -%>
      set $.dynadir = "debug-runtime-metrics";
      action(
        type="omfile"
        template="RSYSLOG_DebugFormat"
        dynafile="tpl_dynafile"
      )
      <%- end -%>
      call rs_funnel_metrics
    }
  }
}

############################
## RULESET CUSTOM METRICS ##
############################

# Parse custom events as metrics and increment internal counters.
ruleset(name="rs_custom_metrics" queue.size="1000") {
  action(type="mmfields" separator=" " jsonRoot=".fields")
  foreach ($.field in $.fields) do {
    if ($.field!value startswith "event=") then {
      set $.type = field($.field!value, 61, 2);
    } else if ($.field!value startswith "context=") then {
      set $.context = field($.field!value, 61, 2);
    } else if ($.field!value startswith "status=") then {
      set $.status = field($.field!value, 61, 2);
    }
  }

  # Default context and status if not set
  if (strlen($.context) == 0) then {
    set $.context = "-";
  }
  if (strlen($.status) == 0) then {
    set $.status = "success";
  }

  set $.custom_tags = exec_template("tpl_custom_event_tags");
  set $.inc = dyn_inc("custom.events.total", $.custom_tags);
  <%- if @data["log_drain"]["debug"] -%>
  set $.dynadir = "debug-custom-events";
  action(
    type="omfile"
    template="RSYSLOG_DebugFormat"
    dynafile="tpl_dynafile"
  )
  <%- end -%>
}

##################
## RULESET LOGS ##
##################

# Parse and publish Heroku logs.
ruleset(name="rs_logs") {
  # Count.
  set $.type = "log";
  call rs_count_events

  # Build fields for event-flatten v2
  set $!data!event = $msg;
  set $.format = "text";

  # Try to parse $msg as JSON.
  if ($msg startswith "{") then {
    action(type="mmjsonparse" cookie="" container="$.jsonmsg")
    if ($parsesuccess == "OK") then {
      reset $!data!event = $.jsonmsg;
      reset $.format = "json";
    }
  }

  # Format source field as "heroku.<source>" to identify logs coming from Heroku in Splunk.
  # e.g. "heroku.heroku" for platform/router logs, "heroku.app" for app logs, and "heroku.token" for user API actions.
  set $!data!source = "heroku." & $.source;
  set $!data!sourcetype = $.sourcetype;    # No sourcetype currently.
  set $!data!hostname = $.app_id; # App id, "host", or "heroku", depending on the type of app.
  set $!data!agent_timestamp = exec_template("tpl_timestamp_rfc3339");

  # Tags
  set $!data!tags_schema_id = "any6:1";
  set $!data!tags!dyno = $.dyno; # The dyno name, e.g. "web.1"
  set $!data!tags!app = $.app;
  set $!data!tags!space = $.space;
  set $!data!tags!region = $.region;
  set $!data!tags!severity = $syslogseverity-text;
  set $!data!tags!uuid = "<%= @data["log_drain"]["uuid"] %>";

  # Owner
  set $!data!owner!service = $.service;
  set $!data!owner!team = $.team;
  set $!data!owner!environment = $.environment;
  set $!data!owner!cloud = $.cloud;

  <%- if @data["log_drain"]["filter_dnr_logs"] -%>
  # Filter DNR logs from a few sources:
  # - Heroku system and API logs.
  # - JSON formatted app logs with "logRecordType=(Authentication|Verification)".
  # - App logs matching a user-provided filter expression.
  set $.dnr = "false";

  if ($.source == "heroku") or ($.source == "app" and $.dyno == "api") then {
    reset $.dnr = "true";
  }

  if ($.format == "json") and (tolower($.jsonmsg!logRecordType) == ["authentication", "verification"]) then {
    reset $.dnr = "true";
  } else if ($.format == "json") and ($.jsonmsg!logRecordType == "") then {
    unset $.jsonmsg!logRecordType;
  }

  <%- if @data["log_drain"]["filter_dnr_logs_expression"] -%>
  if (<%= @data["log_drain"]["filter_dnr_logs_expression"] %>) then {
    reset $.dnr = "true";
  }
  <%- end -%>

  if ($.dnr == "true") then {
    set $!data!facility = $syslogfacility-text;
    set $!data!priority = $syslogpriority-text;
    set $!data!uuid = $uuid;
    set $!data!dnr!environment = $.business_unit;

    call rs_funnel_dnr_logs
  }
  <%- end -%>

  call rs_funnel_logs
}

###################
## RULESET STATS ##
###################

# Publish Heroku event counts and handle rsyslog internal stats/logs.
ruleset(name="rs_stats" queue.size="1000") {
  set $.inc = dyn_inc("heroku", "metadata.info");
  # Setting format="json" in impstats produces the following JSON, no @cee cookie.
  # { "name": "rs_demux", "origin": "core.queue", "size": 0, "enqueued": 1795, "full": 0,
  #   "discarded.full": 0, "discarded.nf": 0, "maxqsize": 10 }
  action(
    type="mmjsonparse"
    name="action_stats_mmjsonparse"
    cookie=""
    container="$!stats"
  )

  # push out impstats stream
   action(
    type="omprog"
    name="action_impstats_to_json_script"
    template="tpl_impstats_json_record"
    binary="/usr/share/rsyslog/plugins/impstats_to_json.py --log-file=stdout --impstats-file=/tmp/impstats.json --save-interval-secs=120 --allow-list=main_Q,resource_usage,omhttp,action_omhttp_funnel_logs,action_omhttp_funnel_dnr_logs,action_omhttp_funnel_metrics"
    queue.type="LinkedList"
    queue.saveOnShutdown="off"
    queue.workerThreads="1"
    action.resumeInterval="5"
    killUnresponsive="on"
    forceSingleInstance="on"
  )


  # Log these stats events.
  if ($!stats!origin == ["core.queue", "core.action", "impstats", "imhttp", "omhttp", "dynstats.bucket", "dynstats"]) then {
    call rs_logger
  }

  # Argus service.
  set $!metric_service = "heroku-drain";

  # Timestamp.
  set $!metric_timestamp = parse_time($timestamp);

  set $.app_id = "<%= @data["heroku"]["app_id"] %>";
  set $.source = "app";
  set $.dyno = "<%= @data["heroku"]["dyno"] %>";

  # Drain tags.
  set $!metric_tags!source = $.source;
  set $!metric_tags!dyno = $.dyno;
  set $!metric_tags!device = $.app_id;
  set $!metric_tags!app = "<%= @data["heroku"]["app_name"] %>";
  set $!metric_tags!space = lookup("heroku_spaces", $.app_id);
  set $!metric_tags!region = lookup("heroku_regions", $.app_id);
  set $!metric_tags!uuid = "<%= @data["log_drain"]["uuid"] %>";

  # Parsing either dynstats bucket or plugin.
  # bucket:   { "name": "heroku", "origin": "dynstats.bucket", "values": { "metadata.info": 16 } }
  # imhttp:   { "name": "imhttp", "origin": "imhttp", "submitted": 26, "failed": 0, "discarded": 0 }
  # impstats: { "name": "resource-usage", "origin": "impstats", "utime": 32000, "stime": 44000, "maxrss": 11948,
  #             "minflt": 2232, "majflt": 0, "inblock": 0, "oublock": 1360, "nvcsw": 2514, "nivcsw": 135, "openfiles": 14 }
  if ($!stats!origin == "dynstats.bucket") then {
    foreach ($.bucket in $!stats!values) do {
      if ($!stats!name == "app.events.total") then {
        # Set a tag that is unique within our app id (uuid) tag.
        # Allows multiple drain dynos to produce metrics about the same app dyno.
        set $!metric_tags!drain_dyno = "<%= @data["heroku"]["dyno"] %>";

        set $.event_app_id = field($.bucket!key, 124, 1);

        # Event tags.
        reset $!metric_tags!source = field($.bucket!key, 124, 2);
        reset $!metric_tags!dyno = field($.bucket!key, 124, 3);
        reset $!metric_tags!type = field($.bucket!key, 124, 4);

        reset $!metric_tags!device = $.event_app_id;
        reset $!metric_tags!app = lookup("heroku_apps", $.event_app_id);
        reset $!metric_tags!space = lookup("heroku_spaces", $.event_app_id);
        reset $!metric_tags!region = lookup("heroku_regions", $.event_app_id);

        # override if exist in bucket!key
        set $._app = field($.bucket!key, 124, 5);
        if $._app != "-" then {
          reset $!metric_tags!app = $._app;
        }
        set $._space = field($.bucket!key, 124, 6);
        if $._space != "-" then {
          reset $!metric_tags!space = $._space;
        }
        set $._team = field($.bucket!key, 124, 8);
        if $._team != "-" then {
          reset $!metric_tags!team = $._team;
        }
        set $._service = field($.bucket!key, 124, 9);
        if $._service != "-" then {
          reset $!metric_tags!service = $._service;
        }

        # Metric: "heroku.app.events.total=<value>".
        set $!metric_name = "heroku." & $!stats!name;
        set $!metric_value = $.bucket!value;
        } else if ($!stats!name == "custom.events.total") then {
          # Format: "<app-id>|<dyno>|<type>|<context>|<status>|<app>|<space>|<region>|<team>|<service>".

          # Set drain dyno tag.
          set $!metric_tags!drain_dyno = "<%= ENV["DYNO"] %>";

          set $.event_app_id = field($.bucket!key, 124, 1);

          # Router event tags.
          reset $!metric_tags!source = "heroku";
          reset $!metric_tags!dyno   = field($.bucket!key, 124, 2);

          set $!metric_tags!event    = field($.bucket!key, 124, 3);
          set $!metric_tags!context = field($.bucket!key, 124, 4);
          set $!metric_tags!status  = field($.bucket!key, 124, 5);

          reset $!metric_tags!device = $.event_app_id;
          reset $!metric_tags!app = lookup("heroku_apps", $.event_app_id);
          reset $!metric_tags!space = lookup("heroku_spaces", $.event_app_id);
          reset $!metric_tags!region = lookup("heroku_regions", $.event_app_id);

          # override if exist in bucket!key
          set $._app = field($.bucket!key, 124, 6);
          if $._app != "-" then {
            reset $!metric_tags!app = $._app;
          }
          set $._space = field($.bucket!key, 124, 7);
          if $._space != "-" then {
            reset $!metric_tags!space = $._space;
          }
          set $._region = field($.bucket!key, 124, 8);
          if $._region != "-" then {
            reset $!metric_tags!region = $._region;
          }
          set $._team = field($.bucket!key, 124, 9);
          if $._team != "-" then {
            reset $!metric_tags!team = $._team;
          }
          set $._service = field($.bucket!key, 124, 10);
          if $._service != "-" then {
            reset $!metric_tags!service = $._service;
          }

          # Metric: "esx.custom.events.total=<value>".
          set $!metric_name = "esx." & $!stats!name;
          set $!metric_value = $.bucket!value;
      } else if ($!stats!name == "drain.events.total") then {
        reset $!metric_tags!type = $.bucket!key;

        # Metric: "heroku.drain.events.total=<value>".
        set $!metric_name = "heroku." & $!stats!name;
        set $!metric_value = $.bucket!value;
      } else if ($!stats!name == "router.requests.total") then {
        # Set drain dyno tag.
        set $!metric_tags!drain_dyno = "<%= @data["heroku"]["dyno"] %>";

        set $.event_app_id = field($.bucket!key, 124, 1);

        # Router event tags.
        reset $!metric_tags!source = "heroku";
        reset $!metric_tags!dyno   = field($.bucket!key, 124, 2);

        set $!metric_tags!method = field($.bucket!key, 124, 3);
        set $!metric_tags!host   = field($.bucket!key, 124, 4);
        set $!metric_tags!status = field($.bucket!key, 124, 5);
        set $!metric_tags!code   = field($.bucket!key, 124, 6);
        reset $!metric_tags!path   = field($.bucket!key, 124, 12);

        reset $!metric_tags!device = $.event_app_id;
        reset $!metric_tags!app = lookup("heroku_apps", $.event_app_id);
        reset $!metric_tags!space = lookup("heroku_spaces", $.event_app_id);
        reset $!metric_tags!region = lookup("heroku_regions", $.event_app_id);

        # override if exist in bucket!key
        set $._app = field($.bucket!key, 124, 7);
        if $._app != "-" then {
          reset $!metric_tags!app = $._app;
        }
        set $._space = field($.bucket!key, 124, 8);
        if $._space != "-" then {
          reset $!metric_tags!space = $._space;
        }
        set $._region = field($.bucket!key, 124, 9);
        if $._region != "-" then {
          reset $!metric_tags!region = $._region;
        }
        set $._team = field($.bucket!key, 124, 10);
        if $._team != "-" then {
          reset $!metric_tags!team = $._team;
        }
        set $._service = field($.bucket!key, 124, 11);
        if $._service != "-" then {
          reset $!metric_tags!service = $._service;
        }

        # Metric: "heroku.router.requests.total=<value>".
        set $!metric_name = "heroku." & $!stats!name;
        set $!metric_value = $.bucket!value;
      } else if ($!stats!name == "heroku" and $.bucket!key == "metadata.info") then {
        set $!metric_tags!release = "<%= @data["heroku"]["release_version"] %>";
        set $!metric_tags!version = "<%= @data["log_drain"]["version"] %>";
        set $!metric_tags!build = "<%= @data["log_drain"]["build"] %>";

        # Metric: "heroku.metadata.info=1".
        set $!metric_name = "heroku." & $.bucket!key;
        set $!metric_value = 1;
      } else {
        stop
      }

      <%- if @data["log_drain"]["debug"] -%>
      set $.dynadir = "debug-event-metrics";
      action(
        type="omfile"
        template="RSYSLOG_DebugFormat"
        dynafile="tpl_dynafile"
      )
      <%- end -%>
      call rs_funnel_metrics
    }
  }

  <%- if @data["log_drain"]["self_metrics"] -%>
  if ($!stats!origin == ["imhttp", "impstats", "omhttp", "core.action", "core.queue"]) then {
    foreach ($.field in $!stats) do {
      if ($.field!key != ["name", "origin"]) then {
        # Replace "-" -> "_" for consistency with other rsyslog metrics.
        set $!metric_name = "rsyslog.impstats." & replace($!stats!name, "-", "_")  & "." & $.field!key;
        set $!metric_value = $.field!value;

        # Do tag customizations by adding `statsname` as action tag
        # for the following origins:
        if ($!stats!origin == "omhttp" or
            $!stats!origin == "core.action") then {
          reset $!metric_name = "rsyslog.impstats." & $!stats!origin & "." & $.field!key;
          set $!metric_tags!action = $!stats!name;
        } else if ($!stats!origin == "core.queue") then {
          # customize for `core.queue` metric
          reset $!metric_name = "rsyslog.impstats." & $!stats!origin & "." & $.field!key;
          set $!metric_tags!queue = $!stats!name;
        }

        <%- if @data["log_drain"]["debug"] -%>
        set $.dynadir = "debug-self-metrics";
        action(
          type="omfile"
          template="RSYSLOG_DebugFormat"
          dynafile="tpl_dynafile"
        )
        <%- end -%>
        call rs_funnel_metrics
      }
    }
  }
  <%- end -%>
}

ruleset(name="rs_logger") {
  # setup tpl_event_flatten_v2 envelope for rsyslog internal logs
  set $!data!tags_schema_id = "any6:1";
  set $!data!tags!dyno = "<%= @data["heroku"]["dyno"] %>";
  set $!data!tags!app = "<%= @data["heroku"]["app_name"] %>";
  set $!data!source = "heroku.app";
  set $!data!sourcetype = "collection:rsyslog";
  set $!data!hostname = "<%= @data["heroku"]["app_id"] %>";
  set $!data!agent_timestamp = exec_template("tpl_timestamp_rfc3339");
  set $!data!tags!space = lookup("heroku_spaces", $!data!hostname);
  set $!data!tags!region = lookup("heroku_regions", $!data!hostname);
  set $!data!tags!severity = $syslogseverity-text;
  set $!data!tags!uuid = "<%= @data["log_drain"]["uuid"] %>";
  set $!data!owner!service = "collection";
  set $!data!owner!team = "collection";
  set $!data!owner!environment = "<%= @data["log_drain"]["environment"] %>";
  set $!data!owner!cloud = "moncloud";
  set $!data!event = $msg;
  set $.format = "text";
  if ($msg startswith "{") then {
    action(type="mmjsonparse" cookie="" container="$.jsonmsg")
    if ($parsesuccess == "OK") then {
      reset $!data!event = $.jsonmsg;
      reset $.format = "json";
    }
  }

  call rs_stdout
  call rs_funnel_logs
}

# Log anything that gets here, too.
call rs_logger
stop