I have nginx json access log like this, a file that append sample like this to end of file.
{
"time": "2019-11-27T08:23:39+00:00",
"msec": 1574843019.787,
"ip": "91.133.248.69",
"agent_name": "firefox",
"agent_version": "70",
"agent_os": "linux",
"agent_os_version": "",
"agent_category": "pc",
"agent_vendor": "mozilla",
"agent_hash": "5c12a77e",
"geo_country_code": "IR",
"geo_latitude": "35.69610",
"geo_longitude": "51.42310",
"geo_accuracy_radius": "50",
"geo_continent_name": "Asia",
"geo_isp": "Mobile Communication Company of Iran PLC",
"method": "GET",
"host": "sample-host.tld",
"request_uri": "/?mamad=1",
"request_length": "218",
"status": "200",
"bytes_sent": "1777",
"body_bytes_sent": "1672",
"referer": "",
"foreign_referer_host": "",
"request_time": "0.006",
"content_type": "text/html; charset=UTF-8",
"brotli_ratio": "",
"gzip_ratio": "3.44",
"upstream_header_time": "0.004",
"upstream_response_time": "0.004",
"upstream_cache_status": ""
}
And i need to export metrics and tagged to prometheus the write way.
I write telegraf to read and process data:
[[inputs.tail]]
name_override = "nginx_access"
files = ["/tmp/access_logs.log"]
from_beginning = false
json_time_key = "msec"
json_time_format = "unix"
pipe = false
data_format = "json"
json_string_fields = [
"bytes_sent",
"body_bytes_sent",
"upstream_response_time",
"upstream_header_time",
"brotli_ratio",
"gzip_ratio",
"request_length",
"request_time"
]
tag_keys = [
"client_new",
"agent_category",
"agent_hash",
"agent_name",
"agent_os",
"agent_os_version",
"agent_vendor",
"agent_version",
"content_type",
"foreign_referer_host",
"geo_accuracy_radius",
"geo_continent_name",
"geo_country_code",
"geo_isp",
"geo_latitude",
"geo_longitude",
"host",
"ip",
"method",
"referer",
"request_uri",
"status"
]
[[processors.converter]]
namepass = ["nginx_access"]
[processors.converter.fields]
integer = [
"bytes_sent",
"body_bytes_sent",
"request_length",
]
float = [
"brotli_ratio",
"upstream_header_time",
"upstream_response_time",
"gzip_ratio",
"request_time"
]
[[processors.regex]]
namepass = ["nginx_access"]
[[processors.regex.tags]]
key = "content_type"
pattern = "^([^/]+/[^ ;]+).*$"
replacement = "${1}"
## output as exporter
[[outputs.prometheus_client]]
listen = ":9273"
collectors_exclude = ["gocollector", "process"]
And it's work and and metrics will be expose
$ curl -s 127.0.0.1:9273/metrics | grep 'nginx_access'
# HELP nginx_access_body_bytes_sent Telegraf collected metric
# TYPE nginx_access_body_bytes_sent untyped
nginx_access_body_bytes_sent{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 1415
nginx_access_body_bytes_sent{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 1672
# HELP nginx_access_brotli_ratio Telegraf collected metric
# TYPE nginx_access_brotli_ratio untyped
nginx_access_brotli_ratio{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 4.06
# HELP nginx_access_bytes_sent Telegraf collected metric
# TYPE nginx_access_bytes_sent untyped
nginx_access_bytes_sent{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 1519
nginx_access_bytes_sent{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 1777
# HELP nginx_access_gzip_ratio Telegraf collected metric
# TYPE nginx_access_gzip_ratio untyped
nginx_access_gzip_ratio{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 3.44
# HELP nginx_access_request_length Telegraf collected metric
# TYPE nginx_access_request_length untyped
nginx_access_request_length{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 387
nginx_access_request_length{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 218
# HELP nginx_access_request_time Telegraf collected metric
# TYPE nginx_access_request_time untyped
nginx_access_request_time{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.006
nginx_access_request_time{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.005
# HELP nginx_access_upstream_header_time Telegraf collected metric
# TYPE nginx_access_upstream_header_time untyped
nginx_access_upstream_header_time{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.004
nginx_access_upstream_header_time{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.004
# HELP nginx_access_upstream_response_time Telegraf collected metric
# TYPE nginx_access_upstream_response_time untyped
nginx_access_upstream_response_time{agent_category="pc",agent_hash="10995fbc",agent_name="chrome",agent_os="linux",agent_os_version="",agent_vendor="google",agent_version="78",client_new="0",client_uid="68d18ac1",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?arash=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.004
nginx_access_upstream_response_time{agent_category="pc",agent_hash="5c12a77e",agent_name="firefox",agent_os="linux",agent_os_version="",agent_vendor="mozilla",agent_version="70",client_new="0",client_uid="54fde64b",content_type="text/html",foreign_referer_host="",geo_accuracy_radius="50",geo_continent_name="Asia",geo_country_code="IR",geo_isp="Mobile Communication Company of Iran PLC",geo_latitude="35.69610",geo_longitude="51.42310",host="sample-host.tld",ip="91.133.248.69",method="GET",path="/memfs/default.nginx.access.log",referer="",request_uri="/?mamad=1",server_country="IR",server_dc="sample",server_hostname="sample-vm-nginx",server_type="vm",status="200"} 0.004
I need to know is my aprouch is correct, is there any misunderstanding?
And afther that how to collect Query per second(QPS) after these structure type.
