input {
redis {
batch_count => 1 #EVAL命令返回的事件数目
data_type => "list" #logstash redis插件工作方式
key => "filebeat" #监听的键值
host => "127.0.0.1" #redis地址
port => 6379 #redis端口号
#password => "123qwe" #如果有安全认证,此项为密码
db => 0 #redis数据库的编号
threads => 1 #启用线程数量
tags =>"test"
}
}
filter {
if "application" in [type] {
if "transaction" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:system}\|%{DATA:ver}\|%{DATA:envir_ID}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}\|%{DATA:result_info}\|%{DATA:service_ID}\|%{DATA:node_ID}\|%{DATA:node_code}\|%{DATA:last_node_ID}\|%{DATA:last_node_code}\|%{DATA:server_ip}\|%{DATA:server_port}\|%{DATA:mode}\|%{DATA:client_type}\|%{DATA:client_ip}\|%{DATA:client_port}\|%{DATA:client_info}\|%{GREEDYDATA:custm_property}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
if [client_info] { #如果标签存在,则继续解析数据,如果不判断会出现_grokparsefailure
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"client_info" => "(?m)%{DATA:phone}&%{DATA:channel}&%{DATA:internet}&%{DATA:client_ver}&%{DATA:screen}&%{DATA:imei}&%{DATA:imsi}&%{DATA:request_time}&%{DATA:response_time}&%{DATA:brand}&%{GREEDYDATA:city}"
}
}
date{
match => ["request_time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["request_time"]
}
date{
match => ["response_time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["response_time"]
}
}
} else if "chain" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}\|%{DATA:result_info}\|%{DATA:service_ID}\|%{DATA:node_code}\|%{DATA:last_node_ID}\|%{DATA:last_node_code}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "method" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:system}\|%{DATA:ver}\|%{DATA:envir_ID}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
}
}
if "systemInfo" in [type] {
if "z_os" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:hostname}\|%{DATA:host_boot_time}\|%{DATA:uptime}\|%{DATA:sys_info}\|%{DATA:sys_processes}\|%{GREEDYDATA:running_processes}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_host" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:processor_load_1}\|%{DATA:processor_load_5}\|%{DATA:processor_load_15}\|%{DATA:context_switches}\|%{DATA:Interrupts}\|%{DATA:user_time}\|%{DATA:Iowait_time}\|%{DATA:nice_time}\|%{DATA:system_time}\|%{DATA:softirq_time}\|%{DATA:steal_time}\|%{DATA:interrupt_time}\|%{DATA:idle_time}\|%{DATA:available_memory}\|%{DATA:free_swap_space}\|%{DATA:free_swap_space_per}\|%{DATA:total_memory}\|%{DATA:total_swap_space}\|%{DATA:incoming}\|%{DATA:outgoing}\|%{DATA:free_disk}\|%{DATA:free_disk_per}\|%{DATA:free_inodes}\|%{DATA:totall_disk}\|%{GREEDYDATA:used_disk}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_redis" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:used_memory}\|%{DATA:used_memory_peak}\|%{DATA:used_memory_lua}\|%{DATA:mem_fragmentation_ratio}\|%{DATA:connected_clients}\|%{DATA:client_longest_output_list}\|%{DATA:client_biggest_input_buf}\|%{DATA:blocked_clients}\|%{DATA:used_cpu_sys}\|%{DATA:used_cpu_user}\|%{DATA:used_cpu_sys_children}\|%{DATA:used_cpu_user_children}\|%{DATA:total_connections_received}\|%{DATA:total_commands_processed}\|%{DATA:instantaneous_ops_per_sec}\|%{DATA:rejected_connections}\|%{DATA:expired_keys}\|%{DATA:evicted_keys}\|%{DATA:keyspace_hits}\|%{GREEDYDATA:keyspace_misses}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_nginx" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:accepts}\|%{DATA:active}\|%{DATA:reading}\|%{DATA:waiting}\|%{DATA:writing}\|%{DATA:handled}\|%{GREEDYDATA:requests}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
}
}
}
output {
if ("_grokparsefailure" not in [tags]) and ("_groktimeout" not in [tags]) {
if("z_os" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_os"
}
}else if ("z_host" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_host"
}
} else if ("z_redis" == [log_type]) {
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_redis"
}
} else if ("z_nginx" == [log_type]) {
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_nginx"
}
} else if("transaction" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_transaction"
}
}else if ("chain" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_chain"
}
} else if ("method" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_method"
}
}
stdout { codec => rubydebug}
}
}