下载安装包并解压
cd /usr/local wget http://archive.cloudera.com/cdh5/cdh/5/flume-ng-1.6.0-cdh5.7.1.tar.gz tar -xvf flume-ng-1.6.0-cdh5.7.1.tar.gz rm flume-ng-1.6.0-cdh5.7.1.tar.gz mv apache-flume-1.6.0-cdh5.7.1 flume-1.6.0-cdh5.7.1
配置环境变量
cd /usr/local vim .bash_profile export FLUME_HOME=/usr/local/flume-1.6.0-cdh5.7.1 export PATH=$PATH:$FLUME_HOME/bin source .bash_profile
配置flume-env.sh文件
cd flume-1.6.0-cdh5.7.1/conf/ cp flume-env.sh.template flume-env.sh vim flume-env.shexport JAVA_HOME=/usr/local/jdk1.7.0_79
版本验证
flume-ng version
部署
flume最主要的是sink,sources,和
cd /usr/local/flume-1.6.0-cdh5.7.1/conf/vim test.confagent1的组件名称agent1.sources = source1agent1.sinks = sink1agent1.channels = channel1# 指定Flume source(要监听的路径)#(taildir source ---------windows 暂不支持这种source)agent1.sources.source1.type = taildiragent1.sources.source1.positionFile = /flume/account/taildir_position.jsonagent1.sources.source1.filegroups = f1agent1.sources.source1.filegroups.f1 =/root/btc/logs/account/dcp/multiple.logagent1.sources.source1.headers.f1.headerKey1 = value1agent1.sources.source1.fileHeader = trueagent1.sources.source1.type#(exec source ---------)#agent1.sources.source1.type = exec#agent1.sources.source1.command = tail -F /var/log/secure# 指定Flume sinkagent1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSinkagent1.sinks.sink1.topic = test#如果kafka是集群,只需要有逗号分隔agent1.sinks.sink1.brokerList = 192.168.80.110:9092agent1.sinks.sink1.requiredAcks = 1agent1.sinks.sink1.batchSize = 100#自定义sink(写东西进数据库)agent1.sinks.mysqlSink.type =com.us.flume.MySinkagent1.sinks.mysqlSink.hostname=localhostagent1.sinks.mysqlSink.port=3306agent1.sinks.mysqlSink.databaseName=sinktestagent1.sinks.mysqlSink.tableName=testagent1.sinks.mysqlSink.user=rootagent1.sinks.mysqlSink.password=xxxxxx# 指定Flume channel内存channelagent1.channels.channel1.type = memoryagent1.channels.channel1.capacity = 1000agent1.channels.channel1.transactionCapacity = 100# 持久化channel,文件channel#agent1.channels.file_channel.type = file#agent1.channels.file_channel.checkpointDir = /var/log/flume-ng/checkpoint#agent1.channels.file_channel.dataDirs = /var/log/flume-ng/data# 绑定source和sink到channel上agent1.sources.source1.channels = channel1agent1.sinks.sink1.channel = channel1
启动flume命令
#-c 是flume 安装目录下的conf
#-f 是配置文件的绝对路径,可以不在flume的安装目录下
#-n 是配置文件中的agent1 ,只是一个名称
#-Dflume.root.logger=INFO,console是log类型,这里是控制台输出
flume-ng agent -c /usr/local/flume-1.6.0-cdh5.7.1/conf -f /usr/local/flume-1.6.0-cdh5.7.1/conf/test.conf -n agent1 -Dflume.root.logger=INFO,consoleflume-ng.cmd agent -conf ../conf -conf -file ../conf/mysqlSink.conf -name agent1 -property flume.root.logger=INFO,console
flume 更详细的东西可以查看文档,里面有很多种sink,source.我这里只是列举了我所使用的两种