ELK日誌平臺----解耦配置文件
阿新 • • 發佈：2017-07-12
elk日誌分析平臺
/* */ ~]# cat shipper.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/nginx/access_json.log"
codec => "json"
type => "nginx_log"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/Mint.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
syslog {
type => "system-syslog"
host => "192.168.1.87"
port => "514"
}
}
output {
if [type] == "system" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system"
}
}
if [type] == "es-error" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "es-error"
}
}
if [type] == "nginx_log" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "nginx_log"
}
}
if [type] == "system-syslog" {
redis {
host => "192.168.1.87 "
port => "6379"
db => "6"
data_type => "list"
key => "system-syslog"
}
}
}
indexer.conf 配置logstash從redis裏讀取日誌內容輸出到Elasticsearch裏。
[[email protected]本文記錄了三個配置文件：
第一個：all.conf 通過一個配置文件，配置輸入輸出，實例；
第二個：shipper.conf配置logstash收集日誌內容到redis裏；
第三個：indexer.conf配置logstash從redis裏讀取日誌內容輸出到Elasticsearch裏。
第二個跟第三個配置解耦收集日誌
ELK 解耦
logstash ---------->redis ---------->logstash -------->elasticsearch----------->kibana
收集 消息隊列 分析 存儲 顯示
192.168.1.87 192.168.1.87 192.168.1.88 192.168.1.87 192.168.1.87
all.conf 通過一個配置文件，配置輸入輸出，實例；
[[email protected] ~]# cat all.conf
input {
file {
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
file {
path => "/var/log/nginx/access_json.log"
codec => "json"
type => "nginx_log"
start_position => "beginning"
}
file {
path => "/var/log/elasticsearch/Mint.log"
type => "es-error"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
syslog {
type => "system-syslog"
host => "192.168.1.87"
port => "514"
}
}
output {
if [type] == "system" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
if [type] == "es-error" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "es-error-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx_log" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "nginx_log-%{+YYYY.MM.dd}"
}
}
if [type] == "system-syslog" {
elasticsearch {
hosts => ["192.168.1.87:9200"]
index => "system-syslog-%{+YYYY.MM.dd}"
}
}
}
shipper.conf配置logstash收集日誌內容到redis裏；
[[email protected]