This is not so much a question, but a post to share how we are sending our metrics from the Fly supplied Prometheus to Better Stack.
During our evaluation of Better Stack I went down the rabbit hole of trying to figure out how to have Better Stack pull our metrics from the Prometheus, similar to what Grafana can do, or have Vector (aka GitHub - superfly/fly-log-shipper: Ship logs from fly to other providers) push our metrics into Better Stack.
Through trial and error I’ve come up with the follow Vector toml which works well for us.
Hopefully this is helpful to others. And if you have any suggestions on how to improve this, please comment away.
# taken from https://github.com/superfly/fly-log-shipper/pull/59
[sources.better_stack_org_metrics]
type = "prometheus_scrape"
endpoints = ["https://api.fly.io/prometheus/${ORG}/federate"]
query.match = ['{__name__!=""}']
auth.strategy = "bearer"
auth.token = "${ACCESS_TOKEN}"
[transforms.better_stack_prometheus_log_transform]
type = "metric_to_log"
inputs = ["better_stack_org_metrics"]
# we use this to filter out metrics from the log shipper and other apps we don't need to monitor
[transforms.better_stack_prometheus_filter_app]
type = "filter"
inputs = ["better_stack_prometheus_log_transform"]
condition = '''
contains(string!(.tags.app), "our-common-app-prefix")
'''
# taken from https://betterstack.com/docs/logs/prometheus/#2-setup
[transforms.better_stack_prometheus_transform]
type = "remap"
inputs = ["better_stack_prometheus_filter_app"]
source = '''
.dt = del(.timestamp)
string_name = string(.name) ?? ""
if (ends_with(string_name, "_total") || ends_with(string_name, "_sum") || ends_with(string_name, "_counter")) && is_object(.gauge) {
.counter.value = del(.gauge.value)
del(.gauge)
}
'''
[sinks.better_stack_http_sink_metrics]
type = "http"
inputs = ["better_stack_prometheus_transform"]
uri = "https://in.logs.betterstack.com/metrics"
method = "post"
encoding.codec = "json"
auth.strategy = "bearer"
auth.token = "${BETTER_STACK_SOURCE_TOKEN_METRICS}"