春播elk_tomcat_配置实例（k8s）
logstash配置文件
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# ll
total 8
-rw-r--r-- 1 root root 3727 May 18 2017 k8selkpatterns
-rw-r--r-- 1 root root 1526 Jun 8 2017 k8slog-to-elk.conf
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8slog-to-elk.conf
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
file {
path => "/data/tomcat/logs/catalina.*.log"
codec=>multiline {
patterns_dir => "/opt/logstash/pipeline/k8selkpatterns"
pattern => "(^%{TOMCAT_DATESTAMP})|(^%{CATALINA_DATESTAMP})"
negate => true
what => "previous"
}
start_position => beginning
add_field => {"type"=>"catalina"}
add_field => {"deployment"=>"fr-web"}
add_field => {"logfiletag" => "catalina.out"}
}
file {
path => "/data/tomcat/fr/fr.log"
codec=>multiline {
patterns_dir => "/opt/logstash/pipeline/k8selkpatterns"
pattern => "(^%{TOMCAT_DATESTAMP})|(^%{CATALINA_DATESTAMP})|(^%{TIMESTAMP_ISO8601})"
negate => true
what => "previous"
}
add_field => {"type"=>"catalina"}
add_field => {"deployment"=>"fr-web"}
add_field => {"logfiletag" => "fr.log"}
}
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter{
}
output {
kafka {
bootstrap_servers => "mqkafka01.prod.aiwaly.com:9092,mqkafka02.prod.aiwaly.com:9092" #线上kafka broker地址
topic_id => "OPK8SLog" #OPK8SLog 代表k8s日志
codec => plain {
format => "%{type}|%{deployment}|${HOSTNAME}|%{logfiletag}|%{message}|end|"
}
}
}
#output {
# stdout { codec => rubydebug }
# }
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8s
k8selkpatterns k8slog-to-elk.conf
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# cat k8selkpatterns
VACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
JAVAFILE (?:[A-Za-z0-9_. -]+)
#Allow special <init>, <clinit> methods
JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
#Line number is optional in special cases 'Native method' or 'Unknown source'
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
# Java Logs
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
JAVAFILE (?:[A-Za-z0-9_.-]+)
JAVALOGMESSAGE (.*)
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800
TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}
CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}
# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened...
TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>”(?>\\.|[^\\"]+)+”|”"|(?>’(?>\\.|[^\\']+)+’)|”|(?>(?>\\.|[^\]+)+)|`))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
HOST %{HOSTNAME}
IPORHOST (?:%{HOSTNAME}|%{IP})
HOSTPORT (?:%{IPORHOST=~/\./}:%{POSINT})
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (?>/(?>[\w_%!$@:.,-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\
[root@dockerwiki01-ctc-bj-10-254-8-219-centos pipeline]# logstash启动脚本
root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]# cat logstash_restart.sh
#!/bin/bash
########jdk setting############
JAVA_HOME=/opt/logstash/jdk1.7.0_80
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=$JAVA_HOME/lib
export PATH CLASSPATH
#java -version
PIDS=$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep | awk '{print $1}')
if [ -z "$PIDS" ]; then
echo "No logstash to stop"
else
#kill -s TERM $PIDS
kill -9 $PIDS
fi
sleep 3
#command_start="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/kafka-es-OPFLBLog.conf"
#${command_start} > /dev/null &
command_start2="/opt/logstash/logstash-2.4.0/bin/logstash --allow-env -f /opt/logstash/pipeline/k8slog-to-elk.conf"
${command_start2} > /dev/null &logstash健康检查脚本
[root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]# cat chk_logstash.sh
#!/bin/bash
command001="/opt/logstash/logstash_restart.sh"
if [ "$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep |wc -l )" -lt 2 ]
then
${command001}
echo $(date) "logstash restart process is "
echo $(date) "logstash restart" >> /var/log/chklogstash.log
fi
[root@dockerwiki01-ctc-bj-10-254-8-219-centos logstash]##kaifa
[root@dockerwiki02-ctc-bj-10-254-8-xxx-centos pipeline]# cat kafka-es-k8spodlog.conf
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
kafka {
zk_connect => "mqzk01.prod.aiwaly.com:2191,mqzk02.prod.aiwaly.com:2191,mqzk03.prod.aiwaly.com:2191"
topic_id => "OPK8SLog"
group_id => "kafka-es-k8spodlog"
#reset_beginning => false
}
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
grok {
match => { "message" => ["%{DATA:type}\|%{DATA:deployment}\|%{DATA:hostname}\|%{DATA:logfiletag}\|%{GREEDYDATA:messager}\|end\|"] }
remove_field =>["message"]
add_field => [ "day", "%{+dd}" ]
add_field => [ "mouth", "%{+MM}" ]
add_field => [ "year", "%{+YYYY}" ]
}
}
output {
elasticsearch {
hosts => ["10.254.64.xxx:9200","10.254.64.xxx:9200","10.254.64.xxx:9200"]
index => "op-k8slog-%{+YYYY}.%{+MM}.%{+dd}"
document_type=>"op-k8slog-"
idle_flush_time => 10
flush_size => 10000
template => "/opt/logstash/pipeline/k8selk-tomplate.json"
template_name =>"k8selk-tomplate"
template_overwrite => true
}
}
#output {
# stdout { codec => rubydebug }
# }
[root@dockerwiki02-ctc-bj-10-254-8-220-centos pipeline]# cat k8selk-tomplate.json
{
"template" : "k8selk-tomplate",
"settings" : {
"index.refresh_interval" : "10s",
"number_of_shards" : 2
},
"mappings" : {
"_default_" : {
"properties":{
"hostname":{
"type":"string",
"index":"not_analyzed"
},
"deployment":{
"type":"string",
"index":"not_analyzed"
},
"type":{
"type":"string",
"index":"not_analyzed"
},
"logfiletag":{
"type":"string",
"index":"not_analyzed"
},
"day":{
"type":"integer",
"index":"not_analyzed"
},
"mouth":{
"type":"integer",
"index":"not_analyzed"
},
"year":{
"type":"integer",
"index":"not_analyzed"
},
"messager":{
"type":"string",
"index":"analyzed"
}
}
}
}
}
文档更新时间: 2019-06-20 02:57 作者：月影鹏鹏
