春播elk_tomcat_配置实例(k8s)
logstash配置文件

[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# ll
total 8
-rw-r--r-- 1 root root 3727 May 18  2017 k8selkpatterns
-rw-r--r-- 1 root root 1526 Jun  8  2017 k8slog-to-elk.conf
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8slog-to-elk.conf 
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
    file {
        path => "/data/tomcat/logs/catalina.*.log"
        codec=>multiline {
        patterns_dir => "/opt/logstash/pipeline/k8selkpatterns"
        pattern => "(^%{TOMCAT_DATESTAMP})|(^%{CATALINA_DATESTAMP})"
        negate => true
        what => "previous"
        }

        start_position => beginning       
        add_field => {"type"=>"catalina"}
        add_field => {"deployment"=>"fr-web"}
        add_field => {"logfiletag" => "catalina.out"}
    }

    file {
        path => "/data/tomcat/fr/fr.log"
        codec=>multiline {
        patterns_dir => "/opt/logstash/pipeline/k8selkpatterns"
        pattern => "(^%{TOMCAT_DATESTAMP})|(^%{CATALINA_DATESTAMP})|(^%{TIMESTAMP_ISO8601})"
        negate => true
        what => "previous"
        } 
        add_field => {"type"=>"catalina"}
        add_field => {"deployment"=>"fr-web"}
        add_field => {"logfiletag" => "fr.log"}
      }
}
# The filter part of this file is commented out to indicate that it is
# optional.


filter{
}


output {
    kafka {
        bootstrap_servers => "mqkafka01.prod.aiwaly.com:9092,mqkafka02.prod.aiwaly.com:9092" #线上kafka broker地址
        topic_id => "OPK8SLog" #OPK8SLog 代表k8s日志
        codec => plain {

           format => "%{type}|%{deployment}|${HOSTNAME}|%{logfiletag}|%{message}|end|"
        }
    }
}

#output {
#    stdout { codec => rubydebug }
# }
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8s
k8selkpatterns      k8slog-to-elk.conf  
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8selkpatterns 
VACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
JAVAFILE (?:[A-Za-z0-9_. -]+)
#Allow special <init>, <clinit> methods
JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
#Line number is optional in special cases 'Native method' or 'Unknown source'
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
# Java Logs
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
JAVAFILE (?:[A-Za-z0-9_.-]+)
JAVALOGMESSAGE (.*)
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}



USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b

POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>”(?>\\.|[^\\"]+)+”|”"|(?>’(?>\\.|[^\\']+)+’)|”|(?>(?>\\.|[^\]+)+)|`))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}

# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME}
IPORHOST (?:%{HOSTNAME}|%{IP})
HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})

# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# 

logstash启动脚本

root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]# cat logstash_restart.sh 
#!/bin/bash

########jdk setting############
JAVA_HOME=/opt/logstash/jdk1.7.0_80
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=$JAVA_HOME/lib
export PATH CLASSPATH
#java -version




PIDS=$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep | awk '{print $1}')

if [ -z "$PIDS" ]; then
  echo "No logstash  to stop"
else 
   #kill -s TERM $PIDS
   kill -9 $PIDS
fi

sleep 3

#command_start="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/kafka-es-OPFLBLog.conf"
#${command_start} > /dev/null & 

command_start2="/opt/logstash/logstash-2.4.0/bin/logstash --allow-env  -f /opt/logstash/pipeline/k8slog-to-elk.conf"
${command_start2} > /dev/null &

logstash健康检查脚本

[root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]# cat chk_logstash.sh 
#!/bin/bash
command001="/opt/logstash/logstash_restart.sh"
if [ "$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep |wc -l )" -lt 2 ]
then
 ${command001}
 echo $(date) "logstash  restart  process is "
 echo $(date) "logstash restart"  >> /var/log/chklogstash.log
fi
[root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]#

#kaifa

[root@dockerwiki02-ctc-bj-10-254-8-xxx-centos pipeline]# cat kafka-es-k8spodlog.conf 
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
    kafka {
        zk_connect => "mqzk01.prod.aiwaly.com:2191,mqzk02.prod.aiwaly.com:2191,mqzk03.prod.aiwaly.com:2191"
        topic_id => "OPK8SLog"

        group_id => "kafka-es-k8spodlog"
        #reset_beginning => false
    }
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
  grok {
    match => { "message" => ["%{DATA:type}\|%{DATA:deployment}\|%{DATA:hostname}\|%{DATA:logfiletag}\|%{GREEDYDATA:messager}\|end\|"] }
           remove_field =>["message"] 
           add_field => [ "day", "%{+dd}" ]
           add_field => [ "mouth", "%{+MM}" ]
           add_field => [ "year", "%{+YYYY}" ]


  }
}
output {
        elasticsearch {
            hosts => ["10.254.64.xxx:9200","10.254.64.xxx:9200","10.254.64.xxx:9200"]
            index => "op-k8slog-%{+YYYY}.%{+MM}.%{+dd}"
            document_type=>"op-k8slog-"
            idle_flush_time => 10
            flush_size => 10000
            template => "/opt/logstash/pipeline/k8selk-tomplate.json"
            template_name =>"k8selk-tomplate"
            template_overwrite => true
       }
}

#output {
#    stdout { codec => rubydebug }
# }
[root@dockerwiki02-ctc-bj-10-254-8-220-centos pipeline]# cat k8selk-tomplate.json 
{
  "template" : "k8selk-tomplate",
  "settings" : {
    "index.refresh_interval" : "10s",
    "number_of_shards" : 2
  },
  "mappings" : {
    "_default_" : {
        "properties":{
          "hostname":{
              "type":"string",
              "index":"not_analyzed"
          },
          "deployment":{
              "type":"string",
              "index":"not_analyzed"
          },
          "type":{
              "type":"string",
              "index":"not_analyzed"
          },
          "logfiletag":{
              "type":"string",
              "index":"not_analyzed"
          },
          "day":{
              "type":"integer",
              "index":"not_analyzed"
          },
          "mouth":{
              "type":"integer",
              "index":"not_analyzed"
          },

          "year":{
              "type":"integer",
              "index":"not_analyzed"
          },
          "messager":{
              "type":"string",
              "index":"analyzed"
          }

        }
    }
  }
 }
文档更新时间: 2019-06-20 02:57   作者:月影鹏鹏