flume下沉Nginx产生的日志文件到hdfs上

1.将工程打成jar包放到hdp-4虚拟机上

2，配置flume

   在/etc/profile中配置frame

在/apps/flume-1.6.0/conf 下vi frame.logger.properties 在里边写配置信息

# Name the components on this agent

a1.sources = r1  

a1.sinks = k1

a1.channels = c1



# Describe/configure the source

a1.sources.r1.type = spooldir

# 根据需要修改监控日志目录： nginx为：/usr/local/nginx/logs

a1.sources.r1.spoolDir = /usr/local/nginx/logs

a1.sources.r1.fileHeader = true

a1.sources.r1.interceptors = i1

a1.sources.r1.interceptors.i1.type = timestamp



# Describe the sink

a1.sinks.k1.type = hdfs

# 收集到hdfs的地址

a1.sinks.k1.hdfs.path = hdfs://hdp-1:9000/output/flume

a1.sinks.k1.hdfs.writeFormat = Text

a1.sinks.k1.hdfs.fileType = DataStream

a1.sinks.k1.hdfs.rollInterval = 10

a1.sinks.k1.hdfs.rollSize = 0

a1.sinks.k1.hdfs.rollCount = 0

a1.sinks.k1.hdfs.filePrefix = %Y-%m-%d-%H-%M-%S

a1.sinks.k1.hdfs.useLocalTimeStamp = true



# Use a channel which buffers events in file

a1.channels.c1.type = file

a1.channels.c1.checkpointDir = /home/hadoop/flume/checkpoint

a1.channels.c1.dataDirs = /home/hadoop/flume/data



# Bind the source and sink to the channel

a1.sources.r1.channels = c1

a1.sinks.k1.channel = c1

3,配置Nginx

cd /usr/local/nginx/conf/ 修改nginx.conf

启动nginx

4.启动工程 java –jar 工程名

5.页面访问

查看是否在/usr/local/nginx/logs中产生日志文件

确定产生日志文件

6.在hdfs查看是否flume下沉文件

7.完成