how to config Filebeat6 quickly

2018-04-11 19:39:00
环境介绍

系统为Centos6.8,相关软件版本如下：
filebeat-6.2.3
redis-3.0.7
logstash-6.2.3
kibana-6.2.3

架构为前端filebeat 读取nginx日志或其他日志（json格式），输出到中间redis，后端logstash从redis读取并解析

filebeat.yml配置
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#filebeat配置
cat /etc/filebeat/filebeat.yml 
filebeat.config_dir: prospectors.d
filebeat.config.prospectors:
  enabled: true
  path: prospectors.d/*.yml
  reload.enabled: true
  reload.period: 10s 
filebeat.prospectors:
- type: log
  enabled: false
  paths:
    - /var/log/message
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
  reload.period: 10s
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
 
output.file:   #主要用于调试
   path: "/tmp"
   filename: filebeat.out
   number_of_files: 7
   rotate_every_kb: 10000 
   enabled: false   #关闭输出
output.redis:
   hosts: ["192.168.90.147:6379"]
   password: "password"
   key: "filebeat"
   db: 0
   timeout: 60
   max_retires: 3
   bulk_max_size: 4096
   datatype: list
   keys:
     - key: "%{[fields.log_source]}"
       mapping:
         "bash_history": "command-log"
         "nginx"  : "nginx-log"

#bash历史记录
[root@localhost ~]# cat /etc/filebeat/prospectors.d/history.yml 
- type: log
  enabled: true
  paths:
    - /var/log/command.log  
  fields:
    log_source: command-log
#  tags: "bash_history"
  json.keys_under_root: true
  json.add_error_key: true
  json.message_key: TIME
  
#nginx日志配置
[root@localhost ~]# cat /etc/filebeat/prospectors.d/nginx.yml
- type: log
  enabled: true
  paths:
    - /usr/local/openresty/nginx/logs/cms_log.log
  fields:
    log_source: nginx-log
  exclude_lines: ["helo.html"]
JSON文件格式
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#bash_history为json格式，添加到/etc/profile文件
HISTDIR='/var/log/command.log'
if [ ! -f $HISTDIR ];then
touch $HISTDIR
chmod 666 $HISTDIR
fi
export HISTTIMEFORMAT="{\"TIME\":\"%F %T\",\"HOSTNAME\":\"$HOSTNAME\",\"LI\":\"$(who -u am i 2>/dev/null| awk '{print $NF}'|sed -e 's/[()]//g')\",\"LU\":\"$(who am i|awk '{print $1}')\",\"NU\":\"${USER}\",\"CMD\":\""
export PROMPT_COMMAND='history 1|tail -1|sed "s/^[ ]\+[0-9]\+  //"|sed "s/$/\"}/">> /var/log/command.log'

#nginx日志格式为
        log_format json '{"@timestamp":"$time_local",'
                '"source":"nginx147",'
                '"serverAddr":"$server_addr",'
                '"remoteAddr":"$remote_addr",'
                '"remoteUser":"$remote_user",'
                '"size":$body_bytes_sent,'
                '"status":$status,'
                '"time":$request_time,'
                '"method":"$request_method",'
                '"protocol":"$server_protocol",'
                '"url":"$scheme://$host$request_uri",'
                '"host":"$http_host",'
                '"uri":"$uri",'
                '"referer":"$http_referer",'
                '"xforwarded":"$http_x_forwarded_for",'
                '"agent":"$http_user_agent",'
                '"upsTime":"$upstream_response_time",'
                '"sslPro":"$ssl_protocol",'
                '"sslCip":"$ssl_cipher",'
                '"upsStatus":"$upstream_status"}';
logstash配置
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
[root@localhost ~]# cat /etc/logstash/conf.d/history-logstash.conf 
input {
    redis {
        data_type => "list"  
        host => "192.168.90.147"
        port => "6379"
        password => "password"
        key  => "command-log"
    }
}

output {
       if [fields][log_source] == "command-log" { 
      elasticsearch {
          hosts   => ["192.16.90.149:9200"]
          manage_template => false
          action  => "index"
          index   => "command-log-%{+YYYY.MM.dd}"
      }
    }
}

#nginx
[root@localhost ~]# cat /etc/logstash/conf.d/nginx-logstash.conf 
input {
    redis {
        data_type => "list"  
        host => "192.168.90.147"
        port => "6379"
        password => "password"
        key  => "nginx-log"
    }
}
output {

	 if [fields][log_source] == "nginx-log" {
        file {
        path => "/tmp/logs/nginx-%{+YYYY-MM-dd}.log"
        }
   }
   
#     elasticsearch {
#        hosts   => ["192.168.90.149:9200"]
#        action  => "index"
#        index   => "nginx-log-%{+YYYY.MM.dd}"
#    }

#        stdout { codec => rubydebug }
}

备注：
logstash安装完后需要执行以下命令，进行service服务安装
/usr/share/logstash/bin/system-install /etc/logstash/startup.options sysv

ref
Filebeat日志收集器
Elastic测试笔记
Configure the Redis output

您的鼓励是我写作最大的动力

俗话说，投资效率是最好的投资。 如果您感觉我的文章质量不错，读后收获很大，预计能为您提高 10% 的工作效率，不妨小额捐助我一下，让我有动力继续写出更多好文章。