Structured logging with Fly

4

Hi @sgammon

We output JSON formatted logs for our apps, then we use a custom config for the Fly-logs-shipper that parses the log message as JSON then merges the parse JSON with the existing properties supplied by fly.io.

This allows us to replace the severity with our own when the log message comes from our app.

You’ll get some logs coming from other parts of the fly.io infrastructure that will have its own severity so you’ll still want to those messages processed as normal.

Here’s our custom config we use for grafana loki:

[transforms.loki_json]
  type = "remap"
  inputs = ["log_json"]
  source = '''
  .level = .log.level

  if starts_with(.message, "{") ?? false {
    # parse json messages
    structured = object(parse_json(.message) ?? "") ?? { "message": .message }

    # delete message field and merge structured message
    del(.message)
    . |= structured
  } else {
    # parse non-json messages
    structured = object(parse_nginx_log(.message, "combined") ?? "") ?? { "message": .message }

    # delete message field and merge structured message
    del(.message)
    . |= structured
  }
  '''

[sinks.loki]
  type = "loki"
  inputs = ["loki_json"]
  endpoint = "${LOKI_URL}"
  compression = "gzip"
  auth.strategy = "basic"
  auth.user = "${LOKI_USERNAME}"
  auth.password = "${LOKI_PASSWORD}"
  encoding.codec = "json"

  labels.event_provider = "{{event.provider}}"
  labels.fly_region = "{{fly.region}}"
  labels.fly_app_name = "{{fly.app.name}}"
  labels.fly_app_instance = "{{fly.app.instance}}"
  labels.host = "{{host}}"
  labels.level = "{{level}}"

  out_of_order_action = "accept"
5 Likes