1.elasticsearch安裝，叢集部署

Logstash 是開源的伺服器端資料處理管道，能夠同時從多個來源採集資料，轉換資料，然後將資料傳送到您最喜歡的 “儲存庫” 中。（我們的儲存庫當然是 Elasticsearch。）

 yum install elasticsearch-2.3.3.rpm
 rpm -ivh jdk-8u121-linux-x64.rpm
 vim /etc/elasticsearch/elasticsearch.yml #編輯配置檔案
 cluster.name: my-es #叢集名稱
 node.name: server1 #節點名稱
 path.data
 
: /var/lib/elasticsearch/#資料儲存路徑
 path.logs: /var/log/elasticsearch/ #日誌位置
 network.host: 172.25.30.1#網路ip
 http.port: 9200 #訪問埠9200
 bootstrap.mlockall: true #允許記憶體鎖定
 discovery.zen.ping.unicast.hosts: ["server1", "server2", "server3"]#編輯叢集節點域名注意解析

/etc/init.d/elasticsearch start
cd /usr/share/elasticsearch/bin/
cd elk/
 /usr/share
 
/elasticsearch/bin/plugin install                                   file:/root/elk/elasticsearch-head-master.zip #本地路徑安裝外掛
/usr/share/elasticsearch/bin/plugin list #列出支援外掛

新增叢集：節點都要安裝elasticsearch jdk-8u121-linux-x64.rpm
配置節點只需修改node.name,network hosts；
vim /etc/elasticsearch/elasticsearch.yml

/etc/init.d/elasticsearch reload
叢集節點服務配置
vim /etc/elasticsearch/elasticsearch.yml
node.master: true #master
node.data: false #
httpd.enabled: false #
Master配置：

/etc/init.d/elasticsearch reload
其他節點:角色一樣開啟服務配置也相同


[[email protected] elasticsearch]# curl -XDELETE ‘http://172.25.30.1:9200/index’ {“acknowledged”:true}

/opt/logstash/bin/logstash -e ‘input { stdin{ }} output {stdout {}}’
你輸入什麼我輸出什麼

/opt/logstash/bin/logstash -e ‘input { stdin{ }} output { stdout { codec => rubydebug } }’

在某個節點搜尋輸出 根據時間來定位
/opt/logstash/bin/logstash -e ‘input { stdin{ }} output { elasticsearch { hosts => [“172.25.30.1”] index => “logstash-%{+YYYY.MM.dd}”} stdout {codec => rubydebug} }’


我們發現這樣在終端輸入命令不是很方便我們可以把他的輸出格式寫在配置檔案然後在後端執行
cd /etc/logstash/conf.d/
vim es.conf #配置檔案只要以conf.d結尾然後指定就可以被讀到

input {
stdin {}
}

output {
elasticsearch {
hosts => [“172.25.30.1”]
index => “logstash-%{+YYYY.MM.dd}”
}
stdout {
codec => rubydebug
}
}

/opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf


vim es.conf #將資訊記錄在指定檔案裡面:
input {
stdin {}
}

output {
elasticsearch {
hosts => [“172.25.30.1”]
index => “logstash-%{+YYYY.MM.dd}”
}

stdout {
    codec => rubydebug

}
file {
    path => "/tmp/testfile"
    codec => line {format => "custom format: %{message}"}
}

}

/opt/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf

檔案記錄到日誌檔案中：

vim message.conf
input {
file {
path => “/var/log/messages”
start_position => “beginning”
}

}
output {
elasticsearch {

    hosts => ["172.25.30.1"]
    index => "message-%{+YYYY.MM.dd}"
}
stdout {
    codec => rubydebug
}

}

[[email protected] conf.d]# /opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf
在後臺執行重新連線一個終端：
寫入輸入:
[[email protected] ~]# logger test ^10
[[email protected] ~]# cat /var/log/messages



開啟514埠同步server2的日至

當sincedb發生改變server1的後臺執行才會有資料


[[email protected] ~]# vim /etc/rsyslog.conf

. @@172.25.30.1 #通過tcp協議傳輸日誌給172.25.30.1

[[email protected] ~]# /etc/init.d/rsyslog restart
Shutting down system logger: [ OK ]
Starting system logger: [ OK ]
[[email protected] ~]# logger server2 #在server2上面寫入資訊到日誌中會被同步到server1

vim message.coinput {
        file {
                path => "/var/log/elasticsearch/my-es.log"
        start_position => "beginning"
}

}

filter { #input模組--->filter---->多行過濾向上匹配

  multiline {
#type => "type"
pattern => "^\["
negate => true
what => "previous"
}
}

output {
elasticsearch {

hosts => ["172.25.30.1"]
index => "es-%{+YYYY.MM.dd}"
}
stdout {
codec => rubydebug
}
}

opt/logstash/bin/logstash -f /etc/logstash/conf.d/message.conf #制定資料輸出以message.conf檔案格式輸出：

採集apche的日誌登陸資訊：

vim message.conf 
input {
        file {
        path => ["/var/log/httpd/access_log","/var/log/httpd/error_log"]
            start_position => "beginning"
    }

}

#filter {
#  multiline {
##  type => "type"
#   pattern => "^\["
#   negate => true
#   what => "previous"
#   }
#}

output {
    elasticsearch {

        hosts => ["172.25.30.1"]
        index => "apache-%{+YYYY.MM.dd}"
    }
    stdout {
        codec => rubydebug
    }


}

分層採集：

input {
    stdin {}
    }

filter {
  grok { #採集資訊輸出格式
    match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
    }
}

output {
    stdout {
        codec => rubydebug
    }
}

Apche的資料沒有改變啟動採集資訊就採集不到我們找到匹配的資訊之後需要先將sincedb刪除才可以訪問

[root@server1 conf.d]# ls -i /var/log/httpd/access_log #檢視那檔案節點
266527 /var/log/httpd/access_log
[root@server1 conf.d]# ls -i /var/log/httpd/error_log 
266525 /var/log/httpd/error_log
[root@server1 conf.d]# cd 
[root@server1 ~]# cat .sincedb_
.sincedb_452905a167cf4509fd08acb964fdb20c
.sincedb_d5a86a03368aaadc80f9eeaddba3a9f5
.sincedb_ef0edb00900aaa8dcb520b280cb2fb7d
[root@server1 ~]# cat .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d
266527 0 64768 462
266525 0 64768 544
[root@server1 ~]# rm -f .sincedb_ef0edb00900aaa8dcb520b280cb2fb7d

安裝kibana：
Kibana是一個開源的分析與視覺化平臺，設計出來用於和Elasticsearch一起使用的。你可以用kibana搜尋、檢視、互動存放在Elasticsearch索引裡的資料，使用各種不同的圖表、表格、地圖等kibana能夠很輕易地展示高階資料分析與視覺化。
Kibana讓我們理解大量資料變得很容易。它簡單、基於瀏覽器的介面使你能快速建立和分享實時展現Elasticsearch查詢變化的動態儀表盤。安裝Kibana非常快，你可以在幾分鐘之內安裝和開始探索你的Elasticsearch索引資料—-—-不需要寫任何程式碼，沒有其他基礎軟體依賴。

[root@server3 ~]# rpm -ivh kibana-4.5.1-1.x86_64.rpm 
Preparing...                ########################################### [100%]
   1:kibana                 ########################################### [100%]
[root@server3 ~]# vim /opt/kibana/config/kibana.yml 
[root@server3 ~]# /etc/init.d/kibana start
kibana started
[root@server3 ~]# netstat -antlp | grep :5601
tcp        0      0 0.0.0.0:5601                0.0.0.0:*                   LISTEN      2324/node        

進行訪問測試kibana上是否顯示訪問資訊：
[[email protected] ~]$ ab -c 1 -n 10 http://172.25.30.1/index.html
vim /etc/logstash/conf.d/nginx.conf
input {
file {
path => “/var/log/nginx/access.log”
start_position => “beginning”
}
}

filter {
grok {
match => { “message” => “%{COMBINEDAPACHELOG} %{QS:x_forwarded_for}” }
}
}

output {
redis {
host => [“172.25.30.2”] #訪問host其實是redis
port => 6379
data_type => “list”
key => “logstash:redis”
}
stdout {
codec => rubydebug
}
}
scp es.conf server2:/etc/logstash/conf.d/
Server2:redis—->節藕

 tar zxf redis-3.0.6.tar.gz 
 cd redis-3.0.6
 yum install gcc y
 make
 make install
 cd utils/
 ./install_server.sh 
  netstat -antlp | grep 6379
cd /etc/logstash/conf.d/
vim es.conf 
~~~~
input {
    redis{
        host => "172.25.30.2"
        port => 6379
        data_type => "list"
        key => "logstash:redis"

    }
}
output {
    elasticsearch {
        hosts => ["172.25.30.1"]
        index => "nginx-%{+YYYY.MM.dd}"

    }   
}

/etc/init.d/logstash start

Visualiza:
最終我們結合kibana將資料呈現在大螢幕上，彙總分析，該web 伺服器的訪問量，訪問排行榜以及負責人資訊。
1 .nginx客戶訪問排行榜 Vertical bar chart X-bar表示客戶端 ip，Y-bar表示client訪問次數
聯絡人 Markdown weight

3.訪問量 Data table
4.新增到 nginxDashboard