logstash nginx日志收集配置
[root@blk08-ctc-bj-10-254-64-132 ~]# cat /opt/logstash/logstash_restart.sh
#!/bin/bash
########jdk setting############
JAVA_HOME=/opt/logstash/jdk1.7.0_80
export JAVA_HOME
PATH=$JAVA_HOME/bin:$PATH
CLASSPATH=$JAVA_HOME/lib
export PATH CLASSPATH
#java -version
PIDS=$(ps ax | grep -i '/opt/logstash/logstash-2.4.0' | grep java | grep -v grep | awk '{print $1}')
if [ -z "$PIDS" ]; then
echo "No logstash to stop"
else
#kill -s TERM $PIDS
kill -9 $PIDS
fi
sleep 3
command_start="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/lb-nginx-kafka.conf"
${command_start} > /dev/null &
#command_start2="/opt/logstash/logstash-2.4.0/bin/logstash -f /opt/logstash/pipeline/applogs-kafka.conf"
#${command_start2} > /dev/null &
[root@SYSDRA-10-254-64-216-cetnos pipeline]# cat kafka-es-OPFLBLog.conf
# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input {
kafka {
zk_connect => "mqzk01.prod.chunbo.com:2191,mqzk02.prod.chunbo.com:2191,mqzk03.prod.chunbo.com:2191"
topic_id => "OPFLBLog"
#topic_id => "FENginxLog"
group_id => "kafka-es-opflblog"
#reset_beginning => false
}
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
grok {
match => { "message" => "%{DATA:host}\|%{DATA:server_addr}\|%{DATA:remote_addr}\|%{DATA:request_time}\|-\|%{DATA:remote_user}\|%{DATA:msec}\|\[%{DATA:time_local}\]\|%{DATA:request_method}\|%{DATA:request_uri}\|%{DATA:status}\|%{DATA:body_bytes_sent}\|%{DATA:http_referer}\|\"%{DATA:http_user_agent}\"\|%{DATA:http_x_forwarded_for}\|%{DATA:cookie_cb_csd}\|%{DATA:cookie_cb_cba_device_id}\|%{DATA:cookie_cb_user}\|%{DATA:cookie_cb_province_id}\|%{DATA:cookie_cb_address_city}\|%{DATA:cookie_cb_site_id}\|%{DATA:request_body}\|%{DATA:upstream_addr}\|-\|%{GREEDYDATA:msg}\|" }
}
grok {
match => {
"time_local"=>"%{MONTHDAY:day}/%{DATA:month}/%{YEAR:year}:%{DATA:otherdata}"
}
}
if [month] == "Jan"{
mutate {
replace => { "month" => "01" }
}
}
if [month] == "Feb"{
mutate {
replace => { "month" => "02" }
}
}
if [month] == "Mar"{
mutate {
replace => { "month" => "03" }
}
}
if [month] == "Apr"{
mutate {
replace => { "month" => "04" }
}
}
if [month] == "May"{
mutate {
replace => { "month" => "05" }
}
}
if [month] == "Jun"{
mutate {
replace => { "month" => "06" }
}
}
if [month] == "Jul"{
mutate {
replace => { "month" => "07" }
}
}
if [month] == "Aug"{
mutate {
replace => { "month" => "08" }
}
}
if [month] == "Sep"{
mutate {
replace => { "month" => "09" }
}
}
if [month] == "Oct"{
mutate {
replace => { "month" => "10" }
}
}
if [month] == "Nov"{
mutate {
replace => { "month" => "11" }
}
}
if [month] == "Dec"{
mutate {
replace => { "month" => "12" }
}
}
date {
match => [ "time_local", "dd/MMM/yyyy:HH:mm:ss Z" ]
target => "time_local"
}
}
output {
elasticsearch {
hosts => ["10.254.64.226:9200","10.254.64.227:9200","10.254.64.228:9200"]
index => "op-flb-nginx-%{year}.%{month}.%{day}"
document_type=>"lb-access-log"
idle_flush_time => 10
flush_size => 10000
template => "/opt/logstash/pipeline/es-lbnginx-template.json"
template_name =>"lb-nginx-template"
template_overwrite => true
}
}
[root@SYSDRA-10-254-64-216-cetnos pipeline]# cat es-lbnginx-template.json
{
"template" : "op-nginx-*",
"settings" : {
"index.refresh_interval" : "10s",
"number_of_shards" : 3
},
"mappings" : {
"_default_" : {
"properties":{
"host":{
"type":"string",
"index":"not_analyzed"
},
"server_addr":{
"type":"string",
"index":"not_analyzed"
},
"remote_addr":{
"type":"string",
"index":"not_analyzed"
},
"request_time":{
"type":"double",
"index":"not_analyzed"
},
"remote_user":{
"type":"string",
"index":"not_analyzed"
},
"msec":{
"type":"double",
"index":"not_analyzed"
},
"request_method":{
"type":"string",
"index":"not_analyzed"
},
"request_uri":{
"type":"string",
"index":"analyzed",
"analyzer":"general_spliter"
},
"status":{
"type":"integer",
"index":"not_analyzed"
},
"body_bytes_sent":{
"type":"long",
"index":"not_analyzed"
},
"http_referer":{
"type":"string",
"index":"analyzed"
},
"http_user_agent":{
"type":"string",
"index":"analyzed"
},
"http_x_forwarded_for":{
"type":"string",
"index":"analyzed"
},
"request_body":{
"type":"string",
"index":"analyzed",
"analyzer":"general_spliter"
},
"member_id":{
"type":"string",
"index":"not_analyzed"
},
"upstream_addr":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_cba_device_id":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_csd":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_user":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_province_id":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_address_city":{
"type":"string",
"index":"analyzed"
},
"cookie_cb_site_id":{
"type":"string",
"index":"analyzed"
}
}
}
}
}
文档更新时间: 2019-06-20 02:57 作者:月影鹏鹏