Nginx1.9+LuaJIT+Kafka的点播监控系统实战
安装前的准备
Nginx1.9.9
LuaJIT-2.0.4
lua-nginx-module-0.10.2
ngx_devel_kit-0.3.0rc1
lua-resty-kafka
kafka_2.11-0.9.0.1
参考的这几篇文章方成此方案
基于Lua+Kafka+Heka的Nginx Log实时监控系统
提供一个相关软件和详细配置说明的网盘下载地址http://pan.baidu.com/s/1c2LmX36
1 安装LuaJIT
下载http://luajit.org/download.html
http://luajit.org/install.html
命令如下：
tar zxf LuaJIT-2.0.4.tar.gz
cd LuaJIT-2.0.4
make PREFIX=/usr/local/LuaJIT
make install PREFIX=/usr/local/LuaJIT
echo "/usr/local/LuaJIT/lib" > /etc/ld.so.conf.d/usr_local_luajit_lib.conf
ldconfig
#注意环境变量!
export LUAJIT_LIB=/usr/local/LuaJIT/lib
export LUAJIT_INC=/usr/local/LuaJIT/include/luajit-2.0
2 安装lua-nginx-module
https://github.com/openresty/lua-nginx-module/tags
cd /usr/local
tar zxvf lua-nginx-module-0.10.2.tar.gz
3 安装ngx_devel_kit
https://github.com/simpl/ngx_devel_kit/tags
http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
cd /usr/local
tar zxvf ngx_devel_kit-0.3.0rc1.tar.gz
4 安装编译Nginx
http://17173ops.com/2013/11/01/17173-ngx-lua-manual.shtml
给Nginx添加下面的参数，如果已经安装了Nginx则用
nginx -V
--add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1
如：
cd /usr/local/nginx-1.9.9
./configure --prefix=/usr/local/nginxnew/ --with-http_stub_status_module --with-http_ssl_module --with-http_realip_module --add-module=/root/install/ngx_log_if-master --add-module=/usr/local/lua-nginx-module-0.10.2 --add-module=/usr/local/ngx_devel_kit-0.3.0rc1
5 lua插件lua-resty-kafka
https://github.com/doujiang24/lua-resty-kafka
mkdir /usr/local/lua
上传lua-cjson-2.1.0.3.tar.gz到/usr/local/lua
上传lua-resty-kafka到/usr/local/lua
6 lua插件cjson
http://www.kyne.com.au/~mark/software/lua-cjson-manual.html
cd /usr/local/lua
tar zxvf lua-cjson-2.1.0.3.tar.gz
7，nginx的配置
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
log_format ht-video '$request|$msec|$http_x_forwarded_for|$http_user_agent|$request_body';
lua_package_path "/usr/local/lua/lua-resty-kafka/lib/?.lua;;";
server{
listen 8080;
server_name 192.168.50.42;
location / {
root /usr/local/nginx/htm;
index index.html;
}
location /ht{
#root html;
#index index.html index.htm;
if ( $request ~ "GET" ) {
#access_log logs/ht-video.log ht-video;
access_log /htlog/ht-video.log ht-video;
}
# proxy_set_header Host $http_host;
# proxy_pass http://localhost:6099;
client_max_body_size 8000M;
#error_page 405 =200 $1;
}
}
server {
listen 6099;
server_name localhost;
location /ht {
root /usr/local/nginx;
index index.html index.htm;
client_max_body_size 8000M;
error_page 405 =200 $1;
}
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
#为了方便调试，关闭了lua_code_cache，如果是生产环境，应该开启它。
lua_code_cache off;
location / {
root html;
index index.html index.htm;
}
location /lua {
default_type 'text/plain';
content_by_lua 'ngx.say("hello, lua")';
}
location = /lua-v {
content_by_lua '
ngx.header.content_type = "text/plain";
if jit then
ngx.say(jit.version)
else
ngx.say(_VERSION)
end
';
}
location /ht3/video/p {
content_by_lua '
ngx.header.content_type = "text/plain";
--local cjson = require "cjson"
--local client = require "resty.kafka.client"
local producer = require "resty.kafka.producer"
local broker_list = {
{ host = "192.168.50.42", port = 9092 }
}
local broker_list_host_ip = {
{ host = "kafka1.livecourse.com", ip = "192.168.50.42" },
}
local key = "key"
--local message = $request|$msec|$remote_addrx|$http_user_agent|$request_body
local myIP = ngx.req.get_headers()["X-Real-IP"]
if myIP == nil then
myIP = ngx.req.get_headers()["x_forwarded_for"]
end
if myIP == nil then
myIP = ngx.var.remote_addr
end
local h = ngx.req.get_headers()
local message = ngx.req.get_method() .. " " .. ngx.var.uri
if ngx.var.args ~= nil then
message = message .. "?" .. ngx.var.args .. "|"
end
message = message .. ngx.now() .. "|"
message = message .. myIP .. "|"
message = message .. h["user-agent"] .. "|"
local bodyData = ngx.req.get_body_data()
if bodyData == nil then
bodyData = "-"
end
message = message .. bodyData
-- this is async producer_type and bp will be reused in the whole nginx worker
local bp = producer:new(broker_list, { producer_type = "async" , broker_list_host_ip = broker_list_host_ip,refresh_interval = 3000})
local ok, err = bp:send("test", key, message)
-- 定义kafka异步生产者
-- 发送日志消息,send第二个参数key,用于kafka路由控制:
-- key为nill(空)时，一段时间向同一partition写入数据
-- 指定key，按照key的hash写入到对应的partition
-- 指定key，按照key的hash写入到对应的partition
if not ok then
--ngx.say("send err:", err)
ngx.say("void(1);");
return
end
ngx.say("void(0);");
--ngx.say("send success, ok:", ok)
';
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
注意配置文件标红的的，第一个标红的为kafka的安装服务器地址和端口，如果有多台则配置多个，第二个标红的test为kafka的topic名称
8 完成
请求Url：http://localhost/ht3/video/p?a=bbbb 就能将日志内容发送到kafka
下一篇： SQL增删改查语句