日志
Windows下 FileBeat+REDIS+Logstash+elasticserach 日志数据分析
|
准备工作：
filebeat下载地址:https://www.elastic.co/downloads/beats/filebeat
redis下载地址 : http://download.redis.io/releases/redis-3.2.7.tar.gz
Logstash下载地址: https://download.elastic.co/logstash/logstash/logstash-2.0.0.zip
搜索
复制
Elasticsearch下载地址: https://www.elastic.co/downloads/elasticsearch
流程： FileBeat读取数据文件，把数据写Redis内，然后Logstash读取Redis数据，对数据进行处理，写入到Elasticsearch。
数据入到Elasticsearch才好进行数据的读取（JAVA代码处理。下次日志展示）
前提条件，本地JAVA_HOME需要配置OK。
在Windows下，只需要把上述下载的包解压即可。
需要做的主要任务是针对FileBeat的配置文件修改，以及Logstatsh 配置文件修改。
修改完配置文件后启动。
filebeat启动：cmd 进入filebeat目录下，启动filebeat.exe 即可。
redis启动：redis-service 。
Logstash启动：cmd进入logstatsh目录BIN下 logstash -f ../config/logstash.conf 即可。
Elasticsearch启动：Elasticsearch目录bin下的 Elasticsearch.bat 启动。
fileBeat 每次读取完文件，不会再次读取名字相同的文件数据。读取后的数据日志位置：
filebeat\data 目录下，删除重启后，可以重新读取原来文件数据。
Elasticsearch：数据存储是已Index，type。具体定义学习，可以去搜下这方面的资料。
想到看数据是否读取进去，可以根据logstatsh配置文件写入ES内的index ，和type去查看。或者安装es_head.
直接URL读法方法：
http://127.0.0.1:9200/t_system-*/t_system_host/_search
FileBeat 配置：这里的作用主要是把数据，原封不动的写到Redis内。
注意：yml配置文件的层级结构，否则启动不了。
Logstatsh配置：主要是对数据的转义，转义成你需要的数据类型结构。我这里的转义比较复杂，只有你想不到的。配置文件就已附件形式上传。
下次会主要说明下logstatsh的配置语法。
input {
redis {
batch_count => 1 #EVAL命令返回的事件数目
data_type => "list" #logstash redis插件工作方式
key => "filebeat" #监听的键值
host => "127.0.0.1" #redis地址
port => 6379 #redis端口号
#password => "123qwe" #如果有安全认证，此项为密码
db => 0 #redis数据库的编号
threads => 1 #启用线程数量
tags =>"test"
}
}
filter {
if "application" in [type] {
if "transaction" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:system}\|%{DATA:ver}\|%{DATA:envir_ID}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}\|%{DATA:result_info}\|%{DATA:service_ID}\|%{DATA:node_ID}\|%{DATA:node_code}\|%{DATA:last_node_ID}\|%{DATA:last_node_code}\|%{DATA:server_ip}\|%{DATA:server_port}\|%{DATA:mode}\|%{DATA:client_type}\|%{DATA:client_ip}\|%{DATA:client_port}\|%{DATA:client_info}\|%{GREEDYDATA:custm_property}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
if [client_info] { #如果标签存在，则继续解析数据，如果不判断会出现_grokparsefailure
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"client_info" => "(?m)%{DATA:phone}&%{DATA:channel}&%{DATA:internet}&%{DATA:client_ver}&%{DATA:screen}&%{DATA:imei}&%{DATA:imsi}&%{DATA:request_time}&%{DATA:response_time}&%{DATA:brand}&%{GREEDYDATA:city}"
}
}
date{
match => ["request_time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["request_time"]
}
date{
match => ["response_time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["response_time"]
}
}
} else if "chain" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}\|%{DATA:result_info}\|%{DATA:service_ID}\|%{DATA:node_code}\|%{DATA:last_node_ID}\|%{DATA:last_node_code}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "method" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:log_ver}\|%{DATA:system}\|%{DATA:ver}\|%{DATA:envir_ID}\|%{DATA:time}\|%{NUMBER:interval:float}\|%{DATA:status}\|%{DATA:result_type}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
}
}
if "systemInfo" in [type] {
if "z_os" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:hostname}\|%{DATA:host_boot_time}\|%{DATA:uptime}\|%{DATA:sys_info}\|%{DATA:sys_processes}\|%{GREEDYDATA:running_processes}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_host" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:processor_load_1}\|%{DATA:processor_load_5}\|%{DATA:processor_load_15}\|%{DATA:context_switches}\|%{DATA:Interrupts}\|%{DATA:user_time}\|%{DATA:Iowait_time}\|%{DATA:nice_time}\|%{DATA:system_time}\|%{DATA:softirq_time}\|%{DATA:steal_time}\|%{DATA:interrupt_time}\|%{DATA:idle_time}\|%{DATA:available_memory}\|%{DATA:free_swap_space}\|%{DATA:free_swap_space_per}\|%{DATA:total_memory}\|%{DATA:total_swap_space}\|%{DATA:incoming}\|%{DATA:outgoing}\|%{DATA:free_disk}\|%{DATA:free_disk_per}\|%{DATA:free_inodes}\|%{DATA:totall_disk}\|%{GREEDYDATA:used_disk}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_redis" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:used_memory}\|%{DATA:used_memory_peak}\|%{DATA:used_memory_lua}\|%{DATA:mem_fragmentation_ratio}\|%{DATA:connected_clients}\|%{DATA:client_longest_output_list}\|%{DATA:client_biggest_input_buf}\|%{DATA:blocked_clients}\|%{DATA:used_cpu_sys}\|%{DATA:used_cpu_user}\|%{DATA:used_cpu_sys_children}\|%{DATA:used_cpu_user_children}\|%{DATA:total_connections_received}\|%{DATA:total_commands_processed}\|%{DATA:instantaneous_ops_per_sec}\|%{DATA:rejected_connections}\|%{DATA:expired_keys}\|%{DATA:evicted_keys}\|%{DATA:keyspace_hits}\|%{GREEDYDATA:keyspace_misses}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
} else if "z_nginx" in [message] {
grok {
#patterns_dir => "/usr/local/logstash/patterns"
match => {
"message" => "(?m)%{DATA:log_type}\|%{DATA:host_ip}\|%{DATA:time}\|%{DATA:accepts}\|%{DATA:active}\|%{DATA:reading}\|%{DATA:waiting}\|%{DATA:writing}\|%{DATA:handled}\|%{GREEDYDATA:requests}"
}
remove_field => ["message"]
}
date{
match => ["time","yyyyMMddHHmmssSSS"]
locale => "en"
target => ["time"]
}
}
}
}
output {
if ("_grokparsefailure" not in [tags]) and ("_groktimeout" not in [tags]) {
if("z_os" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_os"
}
}else if ("z_host" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_host"
}
} else if ("z_redis" == [log_type]) {
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_redis"
}
} else if ("z_nginx" == [log_type]) {
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_system-%{+YYYY.MM.dd}"
document_type => "t_system_nginx"
}
} else if("transaction" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_transaction"
}
}else if ("chain" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_chain"
}
} else if ("method" == [log_type]){
elasticsearch{
hosts => ["127.0.0.1:9200"]
index => "t_application-%{+YYYY.MM.dd}"
document_type => "t_application_method"
}
}
stdout { codec => rubydebug}
}
}
