欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识！
您现在的位置是： 首页  >  后端开发

使用flume+kafka+storm构建实时日志分析系统_PHP教程

程序员文章站 2022-04-08 13:12:33
...

使用flume+kafka+storm构建实时日志分析系统

本文只会涉及flume和kafka的结合，kafka和storm的结合可以参考其他博客
1. flume安装使用
下载flume安装包http://www.apache.org/dyn/closer.cgi/flume/1.5.2/apache-flume-1.5.2-bin.tar.gz
解压$ tar -xzvf apache-flume-1.5.2-bin.tar.gz -C /opt/flume
flume配置文件放在conf文件目录下，执行文件放在bin文件目录下。
1）配置flume
进入conf目录将flume-conf.properties.template拷贝一份，并命名为自己需要的名字
$ cp flume-conf.properties.template flume.conf
修改flume.conf的内容，我们使用file sink来接收channel中的数据，channel采用memory channel，source采用exec source，配置文件如下：
    

  1. agent.sources = seqGenSrc
    2. 

  2. agent.channels = memoryChannel
    3. 

  3. agent.sinks = loggerSink
    4. 

  4. 

  5. # For each one of the sources, the type is defined
    6. 

  6. agent.sources.seqGenSrc.type = exec
    7. 

  7. agent.sources.seqGenSrc.command = tail -F /data/mongodata/mongo.log
    8. 

  8. #agent.sources.seqGenSrc.bind = 172.168.49.130
    9. 

  9. 

  10. # The channel can be defined as follows.
    11. 

  11. agent.sources.seqGenSrc.channels = memoryChannel
    12. 

  12. 

  13. # Each sink's type must be defined
    14. 

  14. agent.sinks.loggerSink.type = file_roll
    15. 

  15. agent.sinks.loggerSink.sink.directory = /data/flume
    16. 

  16. 

  17. #Specify the channel the sink should use
    18. 

  18. agent.sinks.loggerSink.channel = memoryChannel
    19. 

  19. 

  20. # Each channel's type is defined.
    21. 

  21. agent.channels.memoryChannel.type = memory
    22. 

  22. 

  23. # Other config values specific to each type of channel(sink or source)
    24. 

  24. # can be defined as well
    25. 

  25. # In this case, it specifies the capacity of the memory channel
    26. 

  26. agent.channels.memoryChannel.capacity = 1000
    27. 

  27. agent.channels.memory4log.transactionCapacity = 100
    28.
2）运行flume agent
切换到bin目录下，运行一下命令：
$ ./flume-ng agent --conf ../conf -f ../conf/flume.conf --n agent -Dflume.root.logger=INFO,console
在/data/flume目录下可以看到生成的日志文件。

2. 结合kafka
由于flume1.5.2没有kafka sink，所以需要自己开发kafka sink
可以参考flume 1.6里面的kafka sink，但是要注意使用的kafka版本，由于有些kafka api不兼容的
这里只提供核心代码，process()内容。

    

  1. Sink.Status status = Status.READY;
    

    2. 


    3. 

  3. Channel ch = getChannel();
    

    4. 

  4. Transaction transaction = null;
    

    5. 

  5. Event event = null;
    

    6. 

  6. String eventTopic = null;
    

    7. 

  7. String eventKey = null;
    

    8. 


    9. 

  9. try {
    

    10. 

  10. transaction = ch.getTransaction();
    

    11. 

  11. transaction.begin();
    

    12. 

  12. messageList.clear();
    

    13. 


    14. 

  14. if (type.equals("sync")) {
    

    15. 

  15. event = ch.take();
    

    16. 


    17. 

  17.  if (event != null) {
    

    18. 

  18.         byte[] tempBody = event.getBody();
    

    19. 

  19.  String eventBody = new String(tempBody,"UTF-8");
    

    20. 

  20.  Map headers = event.getHeaders();
    

    21. 


    22. 

  22.  if ((eventTopic = headers.get(TOPIC_HDR)) == null) {
    

    23. 

  23.           eventTopic = topic;
    

    24. 

  24.  }
    

    25. 


    26. 

  26.         eventKey = headers.get(KEY_HDR);
    

    27. 


    28. 

  28.  if (logger.isDebugEnabled()) {
    

    29. 

  29.  logger.debug("{Event} " + eventTopic + " : " + eventKey + " : "
    

    30. 

  30.  + eventBody);
    

    31. 

  31.  }
    

    32. 

  32.  
    

    33. 

  33.         ProducerData data = new ProducerData
    

    34. 

  34.  (eventTopic, new Message(tempBody));
    

    35. 

  35.  
    

    36. 

  36.  long startTime = System.nanoTime();
    

    37. 

  37.  logger.debug(eventTopic+"++++"+eventBody);
    

    38. 

  38.         producer.send(data);
    

    39. 

  39.  long endTime = System.nanoTime(); 
    40. 

  40.  }
    

    41. 

  41. } else {
    

    42. 

  42. long processedEvents = 0;
    

    43. 

  43. for (; processedEvents 
    44. 

  44. event = ch.take();
    

    45. 


    46. 

  46.  if (event == null) {
    

    47. 

  47.  break;
    

    48. 

  48.  }
    

    49. 


    50. 

  50.  byte[] tempBody = event.getBody();
    

    51. 

  51.  String eventBody = new String(tempBody,"UTF-8");
    

    52. 

  52.  Map headers = event.getHeaders();
    

    53. 


    54. 

  54.  if ((eventTopic = headers.get(TOPIC_HDR)) == null) {
    

    55. 

  55.           eventTopic = topic;
    

    56. 

  56.  }
    

    57. 


    58. 

  58.         eventKey = headers.get(KEY_HDR);
    

    59. 


    60. 

  60.  if (logger.isDebugEnabled()) {
    

    61. 

  61.  logger.debug("{Event} " + eventTopic + " : " + eventKey + " : "
    

    62. 

  62.  + eventBody);
    

    63. 

  63.  logger.debug("event #{}", processedEvents);
    

    64. 

  64.  }
    

    65. 


    66. 

  66.  // create a message and add to buffer
    

    67. 

  67.         ProducerData data = new ProducerData
    

    68. 

  68.  (eventTopic, eventBody);
    

    69. 

  69.         messageList.add(data);
    

    70. 

  70. }
    

    71. 


    72. 

  72. // publish batch and commit.
    

    73. 

  73.  if (processedEvents > 0) {
    

    74. 

  74.  long startTime = System.nanoTime(); 
    75. 

  75.  long endTime = System.nanoTime(); 
    76. 

  76.  }
    

    77. 

  77. }
    

    78. 


    79. 

  79. transaction.commit();
    

    80. 

  80. } catch (Exception ex) {
    

    81. 

  81. String errorMsg = "Failed to publish events";
    

    82. 

  82. logger.error("Failed to publish events", ex);
    

    83. 

  83. status = Status.BACKOFF;
    

    84. 

  84. if (transaction != null) {
    

    85. 

  85. try {
    

    86. 

  86. transaction.rollback(); 
    87. 

  87. } catch (Exception e) {
    

    88. 

  88. logger.error("Transaction rollback failed", e);
    

    89. 

  89. throw Throwables.propagate(e);
    

    90. 

  90. }
    

    91. 

  91. }
    

    92. 

  92. throw new EventDeliveryException(errorMsg, ex);
    

    93. 

  93. } finally {
    

    94. 

  94. if (transaction != null) {
    

    95. 

  95. transaction.close();
    

    96. 

  96. }
    

    97. 

  97. }
    

    98. 


    99. 

  99. return status; 
    100.
下一步，修改flume配置文件，将其中sink部分的配置改成kafka sink，如：

    

  1. producer.sinks.r.type = org.apache.flume.sink.kafka.KafkaSink
    

    2. 

  2. producer.sinks.r.brokerList = bigdata-node00:9092
    

    3. 

  3. producer.sinks.r.requiredAcks = 1
    

    4. 

  4. producer.sinks.r.batchSize = 100
    

    5. 

  5. #producer.sinks.r.kafka.producer.type=async
    

    6. 

  6. #producer.sinks.r.kafka.customer.encoding=UTF-8
    

    7. 

  7. producer.sinks.r.topic = testFlume1
    8.
type指向kafkasink所在的完整路径
下面的参数都是kafka的一系列参数，最重要的是brokerList和topic参数

现在重新启动flume，就可以在kafka的对应topic下查看到对应的日志

www.bkjia.comtruehttp://www.bkjia.com/PHPjc/1109725.htmlTechArticle使用flume+kafka+storm构建实时日志分析系统 本文只会涉及flume和kafka的结合，kafka和storm的结合可以参考其他博客 1. flume安装使用 下载flume安装...
相关标签： android

上一篇： 使用php将某个目录下面的所有文件罗列出来的方法详解_PHP教程

下一篇： 浅析SVN常见问题及解决方法_PHP教程

推荐阅读