teg nginx访问量-错误率-延迟grafana展示

......
http{
......
log_format brown_json  '{"time_local":"$time_iso8601","remote_addr":"$remote_addr","request_status":"$status",'
'"upstream_status":"$upstream_status","request_time":$request_time,"upstream_time":"$upstream_response_time",'
'"body_sent":"$body_bytes_sent","server":"$host","upstream_addr":"$upstream_addr","source_host":"$hostname",'
'"request":"$request","http_referer":"$http_referer","http_user_agent":"$http_user_agent",'
'"http_x_forwarded_for":"$http_x_forwarded_for","globalTicket":"$globalTicket", "remote_port":"$remote_port", "scheme":"$scheme","http_X_Forwarded_Proto":"$http_X_Forwarded_Proto"}';
......
}
server{
......
include upstream/*.conf;
include vhost.d/k8s/*.conf;
......
}

server {
    listen      80;
    server_name api.bthlt.com;

    access_log  /data/log/elk/api.bthlt.com.access.log brown_json;
    error_log  /data/log/nginx/api.bthlt.com.error.log;

    add_header globalticket $globalTicket;
    #proxy_set_header X-Real-IP $remote_addr;
    add_header Access-Control-Allow-Headers globalTicket;
    add_header Access-Control-Max-Age  86400;
    add_header 'Access-Control-Allow-Origin' '*';
    add_header 'Access-Control-Allow-Credentials' 'true';
    add_header 'Access-Control-Allow-Headers' 'Authorization,Content-Type,Accept,Origin,User-Agent,DNT,Cache-Control,X-Mx-ReqToken,X-Requested-With';
    add_header 'Access-Control-Allow-Methods' 'GET,POST,OPTIONS';

    location ~ /\. {
        deny  all;
    }

    location / {
         proxy_redirect off;
         set $globalTicket $pid-$msec-$remote_addr-$request_length-$connection;
         proxy_set_header globalTicket $globalTicket;
         proxy_set_header Host $host:$server_port;
         proxy_http_version 1.1;
         proxy_set_header Connection "";
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_pass http://k8s_node;
     }

}

upstream k8s_node {
      server 10.1.1.2 id=103 weight=10;
      server 10.1.1.5 id=103 weight=10;
      server 10.1.1.7 id=103 weight=10;
      server 10.1.1.20 id=103 weight=10;
      server 10.1.1.22 id=103 weight=10;
      server 10.1.1.23 id=103 weight=10;
      server 10.1.1.25 id=103 weight=10;
      server 10.1.1.26 id=103 weight=10;
      server 10.1.1.29 id=103 weight=10;
    keepalive 100;
    check interval=3000 rise=2 fall=5 timeout=1000 type=tcp;
}

{"time_local":"2019-10-15T13:38:20+08:00","remote_addr":"118.118.118.118","request_status":"200","upstream_status":"200","request_time":0.016,"upstream_time":"0.016","body_sent":"1668","server":"api.bthlt.comm","upstream_addr":"10.10.10.10:80","source_host":"sh-private-nginx-online-01","request":"POST /api3/ec/navigation/page/getPageModule HTTP/1.1","http_referer":"https://bthlt.com/test/test/test-frame.html","http_user_agent":"Mozilla/5.0 (Linux; Android 9; ANE-LX2 Build/HUAWEIANE-L22; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/75.0.3770.143 Mobile Safari/537.36 MicroMessenger/7.0.7.1521(0x27000739) Process/appbrand0 NetType/WIFI Language/zh_CN","http_x_forwarded_for":"103.103.103.103, 103.103.103.103, 118.118.118.118","globalTicket":"28297-1571117900.169-118.118.118.118-1017-1609372566", "remote_port":"29323", "scheme":"http","http_X_Forwarded_Proto":"https"}

input {
  file {
    type => "nginx_access"
    path => ["/data/log/elk/*log"]
    sincedb_path => "/usr/local/logstash-2.4.0/sincedb/shipper"
    codec => json {}
    }
}

filter {
    mutate {
        split => ["request" ," "]
        add_field => ["request_method", "%{request[0]}"]
        add_field => ["request_url", "%{request[1]}"]
        add_field => ["request_schema", "%{request[2]}"]
        remove_field => ["request"]
    }

    mutate {
        split => ["http_x_forwarded_for" ,","]
        add_field => ["real_ip", "%{http_x_forwarded_for[0]}"]
    }

  if [real_ip] != "-" {
    geoip {
        source => "real_ip"
        target => "geoip"
        database => "/usr/local/logstash-2.4.0/geodb/GeoLiteCity.dat"
        add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
        add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    }

    mutate {
        convert => [ "[geoip][coordinates]", "float"]
      }
    }
}

output {
  elasticsearch {
      hosts => ["10.1.1.1:9200","10.1.1.2:9200","10.1.1.3:9200"]
      index => "logstash-saas-router-internet-%{+YYYY.MM.dd}"
      flush_size => 2000
      idle_flush_time => 5
      workers => 24
      codec => json {}
  }
}

日志采集
[program:logstash-nginx]
environment=LS_HEAP_SIZE=2000m
numprocs=1
command=/usr/local/logstash-2.4.0/bin/logstash -f /usr/local/logstash-2.4.0/conf/nginx_access.conf -l /usr/local/logstash-2.4.0/log/nginx_access.log
autostart=true
autorestart=true
stderr_logfile=/usr/local/logstash-2.4.0/log/logstash.nginx.err
stdout_logfile=/usr/local/logstash-2.4.0/log/logstash.nginx.stout
nodaemon=false