log4j+flume+HDFS实现日志存储

Flume配置文件

apache-flume-1.7.0-bin/data/weblog.conf

a1.sources = s1
a1.channels = c1
a1.sinks = k1

a1.sources.s1.type = avro
a1.sources.s1.bind = 0.0.0.0
a1.sources.s1.port = 44444
//配置拦截器
a1.sources.s1.interceptors = i1
a1.sources.s1.interceptors.i1.type = timestamp

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://hadoop01:9000/weblog/reporttime=%Y-%m-%d
a1.sinks.ke.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.rollInterval = 30
a1.sinks.k1.hdfs.rollSize = 0
a1.sinks.k1.hdfs.rollCount = 0

a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

log4j.properties

log4j.rootLogger = info,stdout,flume

log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = %m%n

log4j.appender.flume = org.apache.flume.clients.log4jappender.Log4jAppender
log4j.appender.flume.Hostname = 192.168.197.128
log4j.appender.flume.Port = 44444
log4j.appender.flume.UnsafeMode = true

LogServlet.java

package cn.tedu;

import java.io.IOException;
import java.net.URLDecoder;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.log4j.Logger;



public class LogServlet extends HttpServlet {
	private Logger logger = Logger.getLogger(LogServlet.class);
	
	public void doGet(HttpServletRequest request, HttpServletResponse response)
			throws ServletException, IOException {
		
		String info = URLDecoder.decode(request.getQueryString(),"utf-8");
		
		
		//按照&将属性切分出来
		String[] kvs = info.split("\\&");
		
		//将所有的value拼接在一起，形式：value|value|value
		StringBuffer buffer = new StringBuffer();
		for	(String kv : kvs) {
			//注意：ref是来自页面的属性，可能没有value，为了避免报错，需要进行处理
			String value = kv.split("=").length == 2 ? kv.split("=")[1] : "";
			buffer.append(value + "|");
		}
		//拼接上用户的ip地址
		buffer.append(request.getRemoteAddr());
		System.out.println(buffer);
		logger.info(buffer);
		
		//将数据封装成日志
		//记录日志的级别从低到高依次是：debug->info->warn->error->fatall
		//记录日志必须要有log4j.properties属性配置文件
		//在属性配置文件中，一是指定最低的日志记录级别
		//如果记录的级别小于配置的最低级别，则不会输出日志
		//二是指定记录的日志的输出的目的地
	}

	public void doPost(HttpServletRequest request, HttpServletResponse response)
			throws ServletException, IOException {
		doGet(request, response);
	}

}

JSP随便写一个

启动Tomcat服务

启动Hadoop

启动Flume

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-WA0kGcGb-1598961442313)(C:\Users\Administrator\AppData\Roaming\Typora\typora-user-images\image-20200901195712699.png)]

刷新JSP页面即可获得请求信息，并将请求信息写到日志中，日志文件上传到HDFS

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-lXL3zrBX-1598961442317)(C:\Users\Administrator\AppData\Roaming\Typora\typora-user-images\image-20200901195646251.png)]