一、整體架構

    flume其實就是一個日誌採集agent，在每臺應用伺服器安裝一個flume agent，然後事實採集日誌到HDFS叢集環境儲存，以便後續使用hive或者pig等大資料分析日誌，然後可轉存到mysql供運維查詢或分析使用者行為等。

叢集規劃：Hive只在一個節點（hadoop3）上安裝即可
	主機名		IP				安裝的軟體		執行的程序
	hadoop1	192.168.31.10	jdk、hadoop			        NameNode、DFSZKFailoverController、
	hadoop2	192.168.31.20	jdk、hadoop	、hive、mysql	        NameNode、DFSZKFailoverController、hive、mysql
	hadoop3	192.168.31.30	jdk、hadoop	、flume		        ResourceManager、fulme
	hadoop4	192.168.31.40	jdk、hadoop、zookeeper		DataNode、NodeManager、JournalNode、QuorumPeerMain
	hadoop5	192.168.31.50	jdk、hadoop、zookeeper		DataNode、NodeManager、JournalNode、QuorumPeerMain
	hadoop6	192.168.31.60	jdk、hadoop、zookeeper	        DataNode、NodeManager、JournalNode、QuorumPeerMain
 

 2. 安裝

 tar -zxvf apache-flume-1.5.0-bin.tar.gz -C /usr/cloud/flume

  vim /etc/profile
  
  export JAVA_HOME=/usr/cloud/java/jdk1.6.0_24
  export HADOOP_HOME=/usr/cloud/hadoop/hadoop-2.2.0
  export HBASE_HOME=/usr/cloud/hbase/hbase-0.96.2
  export FLUME_HOME=/usr/cloud/flume/apache-flume-1.5.0
  export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HBASE_HOME/bin:$FLUME_HOME/bin
  
  source /etc/profile
 

        #執行hdfs的配置
	scp $HADOOP_HOME/etc/hadoop/core-site.xml  $FLUME_HOME/conf
	scp $HADOOP_HOME/etc/hadoop/hdfs-site.xml  $FLUME_HOME/conf

sudo vim $FLUME_HOME/conf/flume-env.sh
	
#設定JAVA_HOME
JAVA_HOME=/usr/cloud/java/jdk1.6.0_24

     scp $HADOOP_HOME/share/hadoop/common/hadoop-common-2.2.0.jar                  $FLUME_HOME/lib/
     scp $HADOOP_HOME/share/hadoop/common/lib/hadoop-auth-2.2.0.jar                     $FLUME_HOME/lib/
     scp $HADOOP_HOME/share/hadoop/common/lib/commons-configuration-1.6.jar       $FLUME_HOME/lib/
三、配置flume source，channel，sink ,在conf下新建檔案flume.conf

vim /FLUME_HOME/conf/flume.conf
	 
#定義agent名， source、channel、sink的名稱
a4.sources = r1
a4.channels = c1
a4.sinks = k1

#具體定義source
a4.sources.r1.type = spooldir
a4.sources.r1.spoolDir = /usr/cloud/flume/log

#具體定義channel
a4.channels.c1.type = memory
a4.channels.c1.capacity = 10000
a4.channels.c1.transactionCapacity = 100

#定義攔截器，為訊息新增時間戳
a4.sources.r1.interceptors = i1
a4.sources.r1.interceptors.i1.type = org.apache.flume.interceptor.TimestampInterceptor$Builder


#具體定義sink
a4.sinks.k1.type = hdfs
a4.sinks.k1.hdfs.path = hdfs://ns1/flume/%Y%m%d
a4.sinks.k1.hdfs.filePrefix = events-
a4.sinks.k1.hdfs.fileType = DataStream
#不按照條數生成檔案
a4.sinks.k1.hdfs.rollCount = 0
#HDFS上的檔案達到128M時生成一個檔案
a4.sinks.k1.hdfs.rollSize = 134217728
#HDFS上的檔案達到60秒生成一個檔案
a4.sinks.k1.hdfs.rollInterval = 60

#組裝source、channel、sink
a4.sources.r1.channels = c1
a4.sinks.k1.channel = c1

        #編寫指令碼 start-flume.sh
	vim $FLUME_HOME/bin/start-flume.sh
	
	$FLUME_HOME/bin/flume-ng agent -n a4 -c conf -f $FLUME_HOME/conf/a4.conf -Dflume.root.logger=INFO,console
	 
	#啟動flume
	$FLUME_HOME/bin/start-flume.sh

檢查配置的檔案是否上傳到hdfs

附錄：

   在

/usr/cloud/flume/log在儲存檔案biz.log,內容：
   
ERROR [2016-10-27 14:44:02,482] com.alibaba.scm.common.monitor.log.ScmBizLogger:30 - |<>|traceId==>UUID:3ff0f75f-0ea8-466b-9c15-b96747b8d636<||>bizType==>3000<||>T==>107<||>SUCC==>true<||>bizName==>createInboundOrder_begain<||>IN==>{"appointmentDate":"Oct 27, 2016 2:43:52 PM","sellId":3693710345,"sellName":"天貓國際012","storeCode":"STORE_230095","customsInfoNo":"ASN00105244708","purchaseOrderNo":"PO160822222226","supplierId":300000000001402,"consignOrderNo":"CO160323143157012226","abroad":false,"demandDate":"Oct 27, 2016 2:43:52 PM","saleType":2,"shipType":0,"consignOrderItemDTOList":[{"scItemId":2100771461843,"itemId":2100770298741,"quantity":100,"gmtExpired":"Oct 10, 2016 12:00:00 AM","rejectDays":10,"managerType":3,"rowVersion":0}],"bizType":3000,"rowVersion":0}  <||>bizCode==>CO160323143157012226<||>CM==>InboundOrderClientImpl.createInboundOrder<||>OUT==>{"success":true,"isRetry":false}<||> 
ERROR [2016-10-27 14:44:03,716] com.alibaba.scm.common.monitor.log.ScmBizLogger:30 - |<>|traceId==>UUID:eb51567f-f83a-4dfb-8d38-9b6b58d84021<||>bizType==>3000<||>T==>1359<||>SUCC==>true<||>bizName==>createInboundOrder_end<||>IN==>{"appointmentDate":"Oct 27, 2016 2:43:52 PM","sellId":3693710345,"sellName":"天貓國際012","storeCode":"STORE_230095","customsInfoNo":"ASN00105244708","purchaseOrderNo":"PO160822222226","supplierId":300000000001402,"consignOrderNo":"CO160323143157012226","abroad":false,"demandDate":"Oct 27, 2016 2:43:52 PM","saleType":2,"shipType":0,"consignOrderItemDTOList":[{"scItemId":2100771461843,"itemId":2100770298741,"quantity":100,"gmtExpired":"Oct 10, 2016 12:00:00 AM","rejectDays":10,"managerType":3,"rowVersion":0}],"bizType":3000,"rowVersion":0}  <||>bizCode==>CO160323143157012226<||>CM==>InboundOrderClientImpl.createInboundOrder<||>OUT==>{"success":true,"isRetry":false,"model":{"purchaseOrderNo":"PO160822222226","inboundNo":"IO16102714440300373001","consignOrderNo":"CO160323143157012226","storeCode":"STORE_230095","supplierId":300000000001402,"sendQuantity":100,"preArrival":"Oct 27, 2016 2:43:52 PM","status":10,"demandDate":"Oct 27, 2016 2:43:52 PM","inboundItemDOList":[{"inboundNo":"IO16102714440300373001","inboundId":373001,"scItemId":2100771461843,"itemId":2100770298741,"sendQuantity":100,"status":1,"rejectDays":10,"managerType":3,"gmtExpired":"Oct 10, 2016 12:00:00 AM","storeCode":"STORE_230095","purchaseOrderNo":"PO160822222226","bizType":3000,"attribute":"MXSDUEDATE:20161010","attributeMap":{"MXSDUEDATE":"20161010"}}],"id":373001,"bizType":3000}}<||>