logstash 配置

logstash nginx日志收集配置

[root@blk08-ctc-bj-10-254-64-132 ~]# cat /opt/logstash/logstash_restart.sh
#!/bin/bash

########jdk setting############
JAVA_HOME=/opt/logstash/jdk1.7.0_80
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=$JAVA_HOME/lib
export PATH CLASSPATH
#java -version




PIDS=$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep | awk '{print $1}')



if [ -z "$PIDS" ]; then
  echo "No logstash  to stop"
else 
   #kill -s TERM $PIDS
   kill -9 $PIDS
fi

sleep 3

command_start="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/lb-nginx-kafka.conf"
${command_start} > /dev/null & 

#command_start2="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/applogs-kafka.conf"
#${command_start2} > /dev/null &




[root@SYSDRA-10-254-64-216-cetnos pipeline]# cat kafka-es-OPFLBLog.conf 
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
    kafka {
        zk_connect => "mqzk01.prod.chunbo.com:2191,mqzk02.prod.chunbo.com:2191,mqzk03.prod.chunbo.com:2191"
        topic_id => "OPFLBLog"
        #topic_id => "FENginxLog"
        group_id => "kafka-es-opflblog"
        #reset_beginning => false
    }
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
  grok {
    match => { "message" => "%{DATA:host}\|%{DATA:server_addr}\|%{DATA:remote_addr}\|%{DATA:request_time}\|-\|%{DATA:remote_user}\|%{DATA:msec}\|\[%{DATA:time_local}\]\|%{DATA:request_method}\|%{DATA:request_uri}\|%{DATA:status}\|%{DATA:body_bytes_sent}\|%{DATA:http_referer}\|\"%{DATA:http_user_agent}\"\|%{DATA:http_x_forwarded_for}\|%{DATA:cookie_cb_csd}\|%{DATA:cookie_cb_cba_device_id}\|%{DATA:cookie_cb_user}\|%{DATA:cookie_cb_province_id}\|%{DATA:cookie_cb_address_city}\|%{DATA:cookie_cb_site_id}\|%{DATA:request_body}\|%{DATA:upstream_addr}\|-\|%{GREEDYDATA:msg}\|" }


  }
  grok {
    match => {
        "time_local"=>"%{MONTHDAY:day}/%{DATA:month}/%{YEAR:year}:%{DATA:otherdata}"
    }
  }

  if [month] == "Jan"{
     mutate {
        replace => { "month" => "01" }
    }
  }
  if [month] == "Feb"{
     mutate {
        replace => { "month" => "02" }
    }
  }
  if [month] == "Mar"{
     mutate {
        replace => { "month" => "03" }
    }
  }
  if [month] == "Apr"{
     mutate {
        replace => { "month" => "04" }
    }
  }
  if [month] == "May"{
     mutate {
        replace => { "month" => "05" }
    }
  }
  if [month] == "Jun"{
     mutate {
        replace => { "month" => "06" }
    }
  }
  if [month] == "Jul"{
     mutate {
        replace => { "month" => "07" }
    }
  }
  if [month] == "Aug"{
     mutate {
        replace => { "month" => "08" }
    }
  }
  if [month] == "Sep"{
     mutate {
        replace => { "month" => "09" }
    }
  }
  if [month] == "Oct"{
     mutate {
        replace => { "month" => "10" }
    }
  }
  if [month] == "Nov"{
     mutate {
        replace => { "month" => "11" }
    }
  }
  if [month] == "Dec"{
     mutate {
        replace => { "month" => "12" }
    }
  }

  date {  
     match => [ "time_local", "dd/MMM/yyyy:HH:mm:ss Z" ]  
     target => "time_local"  
  } 
}
output {
        elasticsearch {
            hosts => ["10.254.64.226:9200","10.254.64.227:9200","10.254.64.228:9200"]
            index => "op-flb-nginx-%{year}.%{month}.%{day}"
            document_type=>"lb-access-log"
            idle_flush_time => 10
            flush_size => 10000
            template => "/opt/logstash/pipeline/es-lbnginx-template.json"
            template_name =>"lb-nginx-template"
            template_overwrite => true
       }
}



[root@SYSDRA-10-254-64-216-cetnos pipeline]# cat es-lbnginx-template.json 
{
  "template" : "op-nginx-*",
  "settings" : {
    "index.refresh_interval" : "10s",
    "number_of_shards" : 3
  },
  "mappings" : {
    "_default_" : {
        "properties":{
          "host":{
              "type":"string",
              "index":"not_analyzed"
          },
          "server_addr":{
              "type":"string",
              "index":"not_analyzed"
          },
          "remote_addr":{
              "type":"string",
              "index":"not_analyzed"
          },
          "request_time":{
              "type":"double",
              "index":"not_analyzed"
          },
          "remote_user":{
              "type":"string",
              "index":"not_analyzed"
          },
          "msec":{
              "type":"double",
              "index":"not_analyzed"
          },
          "request_method":{
              "type":"string",
              "index":"not_analyzed"
          },
          "request_uri":{
              "type":"string",
              "index":"analyzed",
              "analyzer":"general_spliter"
          },
          "status":{
              "type":"integer",
              "index":"not_analyzed"
          },
          "body_bytes_sent":{
              "type":"long",
              "index":"not_analyzed"
          },
          "http_referer":{
              "type":"string",
              "index":"analyzed"
          },
          "http_user_agent":{
              "type":"string",
              "index":"analyzed"
          },
          "http_x_forwarded_for":{
              "type":"string",
              "index":"analyzed"
          },
          "request_body":{
              "type":"string",
              "index":"analyzed",
              "analyzer":"general_spliter"
          },
      "member_id":{
              "type":"string",
              "index":"not_analyzed"
          },
          "upstream_addr":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_cba_device_id":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_csd":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_user":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_province_id":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_address_city":{
              "type":"string",
              "index":"analyzed"
          },
          "cookie_cb_site_id":{
              "type":"string",
              "index":"analyzed"
          }


        }
    }
  }
 }