1. 程式人生 > >flume的導日誌資料到hdfs

flume的導日誌資料到hdfs

1.更改配置檔案

#agent名, source、channel、sink的名稱
a1.sources = r1
a1.channels = c1
a1.sinks = k1
#具體定義source
a1.sources.r1.type = spooldir
a1.sources.r1.spoolDir = /root/flume-log

#具體定義channel
a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 100
#具體定義sink
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://172.17.14.209:8020/flume/%Y%m%d
a1.sinks.k1.hdfs.filePrefix = events-
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.useLocalTimeStamp = true
#不按照條數生成檔案
a1.sinks.k1.hdfs.rollCount = 0
#HDFS上的檔案達到128M時生成一個檔案
a1.sinks.k1.hdfs.rollSize = 134217728
#HDFS上的檔案達到60秒生成一個檔案
a1.sinks.k1.hdfs.rollInterval = 30

#組裝source、channel、sink
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

2.把系統組和系統使用者改為root

3.更新配置