php读取大文件示例分享(文件操作类)

<?php
 class lib_file2
 {
  //文件目录
  private $root = '/data/wwwroot/kkpromo/data/';

  //文件后缀
  private $suffix = '.log';

  //文件句柄
  private $handle=null;

  //一次读取文件的最大记录数
  private $limit=40000;

  //每行读取的字节长度
  private $length=1024;

  //开始时间
  private  $starttime=0;

  //内存使用基准点
  private static $startmemory=0;

  //
  private $conn=null;

  //
  private static  $init=null;

  public static function instance()
  {
   self::$startmemory = memory_get_usage(true);

   if(self::$init && is_object(self::$init))
   {
    return self::$init;
   }

   self::$init = new self();

   return self::$init;
  }

  private function __construct(){}

  public  function setroot($root)  
  {
   if(!is_dir($root)) die($root.' root does not exist');
   $this->root = $root;
  }

  public function setsuffix($suffix)
  {
   $this->suffix = $suffix;
  }

  public function setlimit($limit)
  {
   if(!is_numeric($limit)) die($limit.' should be numberic');
   if(intval($limit) > 1000000) die($limit.' should be lower than 1000000');
   $this->limit = intval($limit);
  }

  public function _getfile($date , $appid , $op)
  {
   $filename = rtrim($this->root , '/').directory_separator.$date.directory_separator.$appid.'.'.$op.$this->suffix;
   if(!file_exists($filename))
   {
    die($filename.' file does not exists!');
   }

   if(!is_file($filename))
   {
    die($filename.' file does not exist!');
   }

   if(!is_readable($filename))
   {
    die($filename.'  file access deny!');
   }

   return $filename;
  }

  
  public function closefile($date=null , $appid=null , $op=null)
  {
   if($op && $date && $appid)
   {
    if(is_object($this->handle[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
    {
     unset($this->handle[$date.'_'.$appid.'_'.$op]);
     $this->handle[$date.'_'.$appid.'_'.$op]=null;
    }

    $this->conn[$date.'_'.$appid.'_'.$op]=null;
    $this->handle[$date.'_'.$appid.'_'.$op]=null;
    unset($this->handle[$date.'_'.$appid.'_'.$op]);
   }
   else {
    if(is_array($this->handle) && $this->handle)
    {
     foreach ($this->handle as $key=>$val){
      unset($this->handle[$key]);
      $this->conn[$key]=null;
      $this->handle[$key]=null;
     }
    }
   } 

   return true;
  }

  
  private function _openfile($date , $appid , $op)
  {   
   $this->starttime = microtime(true);
   if(isset($this->conn[$date.'_'.$appid.'_'.$op])  && $this->conn[$date.'_'.$appid.'_'.$op])
   {
    return $this->handle[$date.'_'.$appid.'_'.$op];
   }

   $filename = self::_getfile($date , $appid , $op);
   if(($this->handle[$date.'_'.$appid.'_'.$op] = new splfileobject($filename , 'r'))!=null)
   {
    $this->conn[$date.'_'.$appid.'_'.$op] = true;
    return $this->handle[$date.'_'.$appid.'_'.$op];
   }
   else {
    die('file open failed!');
   }
  }

  
  /**
   * 功能：解析数据
   * 格式：  array('timestamp','mid','data');
   * @param string $data
   * @return boolean|array
   */
  private  function _parsedata($data , $jsonflag=true)
  {
   if(empty($data) || !is_string($data)) return false;
   $result = array(
     'timestamp'=>0,
     'mid'=>0,
     'data'=>array(),
   );

   $data = explode('|', $data);
   if(count($data) < 3 || !is_array($data)) return false;
   $result['timestamp'] = $data[0];
   $result['mid'] = $data[1];
   if($jsonflag)
   {
    $result['data'] = @json_decode($data[2] , true);
    unset($result['mid']);
   }
   if(empty($result['timestamp']) || empty($result['mid'])) return false;

   unset($data);
   return $result;
  }

  
  /**
   * todo:读取单一文件
   * @param string $date： 如（20140327）
   * @param int  $appid: 如（1000,9000）
   * @param string $op：如(show，login , index)
   * @param number $startnum 默认从第一行开始
   * @param number $length 默认到$this->limit 读取的行数
   * @param array $condition:array('mid'=>arrray() , 'ip'=>array() , ...)  过滤条件
   * @param bool $jsonflag：默认为true， 则保留jsondata字段；设为false，则去掉false字段
   * @return array(count , difftime , memory , data)
   */
  public  function readfile($date , $appid , $op , $startnum=0 , $length=0 , $jsonflag=false ,  $condition=array())
  {
   $data['data'] = "";
   $data['count'] = 0;
   $index = $startnum;
   $startnum = empty($startnum) ? 0 : $startnum;
   $length = empty($length) ? $this->limit : $length;

   $handle = self::_openfile($date , $appid , $op );
   $line_number=0;

   if($handle)
   { 
    $handle->seek($startnum);
    $handle->setmaxlinelen($this->length);
    while (intval($line_number) - intval($startnum) < intval($length)-1)
    {
     $tmp = $handle->current();
     if(empty($tmp)) continue;     
     $tmp = self::_parsedata($tmp , $jsonflag);
     $line_number = $handle->key();
     !$jsonflag && $condition= array();        
     if(isset($condition) && $condition)
     {
      $key = array_keys($condition);
      if(in_array($tmp['data'][$key[0]], $condition[$key[0]]))
      {
       $data['count']++;
       $data['data'][$line_number] = $tmp;
      }
     }
     else
     {
      $data['data'][$line_number] = $tmp;
      $data['count']++;
     }

     if(intval($line_number) - intval($startnum) >= intval($length)-1) break;
     unset($tmp);
     $handle->next();
    }
    unset($tmp , $length , $line_number , $condition);
   }

   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';

   return $data;
  }
  

  /**
   * todo:命令行下获取文件总记录数*
   * @param string $date
   * @param int $appid
   * @param string $op
   * @return array
   */
  public  function total_linefile($date, $appid, $op)
  {
   $this->_openfile($date, $appid, $op);
   $file = escapeshellarg($this->_getfile($date, $appid, $op)); // 对命令行参数进行安全转义
   $line = `wc -l  $file`;
   if(preg_match("/(\d{1,})/", $line , $ret)){
    $data['count']=$ret[1];
   }else{
    $data['count']=0;
   }
   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';
   return $data;
  }

  
  /**
   * todo:统计{$data}.{$op}.log记录数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param array $condition
   * @return array
   */
  public function countfile($date , $appid , $op ,$condition=array())
  {
   $data['count'] = 0;
   $handle = self::_openfile($date , $appid , $op );  
   if($handle)
   {
    $handle->setmaxlinelen($this->length);
    while (!$handle->eof())
    {
     $tmp = $handle->current();
     if(empty($tmp)) continue;
     $tmp = self::_parsedata($tmp);
     if($condition && is_array($condition) )
     {      
      $key = array_keys($condition);
      if(isset($tmp['data'][$key[0]]) && $tmp['data'][$key[0]] && in_array($tmp['data'][$key[0]], $condition[$key[0]]))
      {
       $data['count']++;
      }
     }
     else
     {
      $data['count']++;      
     }

     unset($tmp);     
     $handle->next();
    }
   }
   unset($handle , $condition , $tmp , $key , $val);
   self::closefile($date , $appid , $op );

   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';

   return $data;
  }  

  

  /**
   * todo:统计用户数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * * @param bool $jsonflag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
   * @param array $condition
   * @return : array：形如（{"mid":[],"count":2181,"difftime":0.0397667884827,"memory":"3.75 m"}）
   */
  public function countfilemid($date , $appid , $op  ,  $midflag=false , $jsonflag=false,  $condition=array())
  {
   //$count = self::total_linefile($date , $appid , $op );
   $count = self::countfile($date , $appid , $op );
   $index = ceil($count['count'] / $this->limit);
   $result = array('mid'=>array() , 'count'=>0 , 'difftime'=>0 , 'memory'=>0);

   for ($i=0 ; $i<$index ; $i++)
   {
    $startnum = $this->limit*$i;
    $endnum = $this->limit;
    $data = self::readfile($date , $appid , $op  ,  $startnum , $endnum , $jsonflag);
    var_dump($data);exit();

    if($data['data'] && is_array($data['data']))
    {
     foreach ($data['data'] as $arr)
     {
      if($condition && is_array($condition))
      {
       $key = array_keys($condition);
       if(isset($arr['data'][$key[0]])  && (in_array($arr['data'][$key[0]] , $condition[$key]) || empty($condition[$key[0]])))
       {
        $result['mid'][$arr["mid"]] =1;
        $result['count']++;
       }
      }
      else
      {
        $result['mid'][$arr["mid"]] =1; 
        $result['count']++;
      }
      unset($data);
     }
    }
   }
   unset($index , $count , $condition , $data  , $arr);
   self::closefile($date , $appid , $op);

   $result['mid'] = array_keys($result['mid']);  
   if(empty($midflag)) unset($result['mid']); 

   $result['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $result['memory'] = (memory_get_usage(true) - self::$startmemory)/1024/1024 . ' m';
   return $result;
  }

  
  /**
   * todo:跨时间段 统计参加$op用户数据
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param number $day
   * @param bool  $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * @return array 形如（"20140326":{"mid":[],"count":4571,"difftime":0.0806441307068,"memory":"3.75 m"},
  *          "20140325":{"mid":[],"count":2181,"difftime":0.0397667884827,"memory":"3.75 m"}）
   */
  public function getreturnuser($date , $appid , $op , $day=1 , $midflag=false)
  {
   $date_i=0;
   for ($i =0; $i<$day ; $i++){
    $date_i = date('ymd' , strtotime($date)-$i*86400);
    $result[$date_i] = self::countfilemid($date_i , $appid , $op , $midflag); 
   }
   unset($date , $date_i , $appid , $op  , $day);

   return $result;
  }
 }
?>

<?php
 class lib_file1
 {
  //文件目录
  private $root = '/data/wwwroot/kkpromo/data/';

  //文件后缀
  private $suffix = '.log';

  //文件句柄
  private $hander=null;

  //一次读取文件的最大记录数
  private $limit=40000;

  //每行读取的字节长度
  private $length=1024;

  //开始时间
  private  $starttime=0;

  //内存使用基准点
  private static $startmemory=0;

  //
  private $conn=null;

  //
  private static  $init=null;

  public static function instance()
  {
   self::$startmemory = memory_get_usage(true);

   if(self::$init && is_object(self::$init))
   {
    return self::$init;
   }

   self::$init = new self();

   return self::$init;
  }

  private function __construct(){}

  public  function setroot($root)  
  {
   if(!is_dir($root)) die($root.' root does not exist');
   $this->root = $root;
  }

  public function setsuffix($suffix)
  {
   $this->suffix = $suffix;
  }

  public function setlimit($limit)
  {
   if(!is_numeric($limit)) die($limit.' should be numberic');
   if(intval($limit) > 1000000) die($limit.' should be lower than 1000000');
   $this->limit = intval($limit);
  }

  private function _getfile($date , $appid , $op)
  {
   $filename = rtrim($this->root , '/').directory_separator.$date.directory_separator.$appid.'.'.$op.$this->suffix;
   if(!file_exists($filename))
   {
    die($filename.' file does not exists!');
   }

   if(!is_file($filename))
   {
    die($filename.' file does not exist!');
   }

   if(!is_readable($filename))
   {
    die($filename.'  file access deny!');
   }

   return $filename;
  }

  
  public function closefile($date=null , $appid=null , $op=null)
  {
   if($op && $date && $appid)
   {
    if(is_object($this->hander[$date.'_'.$appid.'_'.$op]) || $this->conn[$date.'_'.$appid.'_'.$op])
    {
     fclose($this->hander[$date.'_'.$appid.'_'.$op]);
    }

    $this->conn[$date.'_'.$appid.'_'.$op]=null;
    $this->hander[$date.'_'.$appid.'_'.$op]=null;
   }
   else {
    if(is_array($this->hander) && $this->hander)
    {
     foreach ($this->hander as $key=>$val){
      fclose($this->hander[$key]);
      $this->conn[$key]=null;
      $this->hander[$key]=null;
     }
    }
   } 

   return true;
  }

  
  private function _openfile($date , $appid , $op)
  {   
   $this->starttime = microtime(true);
   if(isset($this->conn[$date.'_'.$appid.'_'.$op])  && $this->conn[$date.'_'.$appid.'_'.$op])
   {
    return $this->hander[$date.'_'.$appid.'_'.$op];
   }

   $filename = self::_getfile($date , $appid , $op);
   if(($this->hander[$date.'_'.$appid.'_'.$op] = fopen($filename, 'r'))!=null)
   {
    $this->conn[$date.'_'.$appid.'_'.$op] = true;
    return $this->hander[$date.'_'.$appid.'_'.$op];
   }
   else {
    die('file open failed!');
   }
  }

  
  /**
   * 功能：解析数据
   * 格式：  array('timestamp','mid','data');
   * @param string $data
   * @return boolean|array
   */
  private  function _parsedata($data)
  {
   if(empty($data) || !is_string($data)) return false;
   $result = array(
     'timestamp'=>0,
     'mid'=>0,
     'data'=>array(),
   );

   $data = explode('|', $data);
   if(count($data) < 3 || !is_array($data)) return false;
   $result['timestamp'] = $data[0];
   $result['mid'] = $data[1];
   $result['data'] = @json_decode($data[2] , true);
   if(empty($result['timestamp']) || empty($result['mid'])) return false;

   unset($data);
   return $result;
  }

  
  /**
   * todo:读取单一文件
   * @param string $date： 如（20140327）
   * @param int  $appid: 如（1000,9000）
   * @param string $op：如(show，login , index)
   * @param number $startnum 默认从第一行开始
   * @param number $endnum 默认到$this->limit结束
   * @param array $condition:array('mid'=>arrray() , 'ip'=>array() , ...)  过滤条件
   * @param bool $jsonflag：默认为true， 则保留jsondata字段；设为false，则去掉false字段
   * @return array(count , difftime , memory , data)
   */
  public  function readfile($date , $appid , $op ,$startnum=0 , $endnum=0 , $jsonflag=false ,  $condition=array())
  {
   $data['data'] = "";
   $data['count'] = 0;
   $index = $startnum;
   $startnum = empty($startnum) ? 0 : $startnum;
   $endnum = empty($endnum) ? $this->limit : $endnum;

   $hander = self::_openfile($date , $appid , $op );
   $tmpindex=0;

   if($hander)
   {
    //!feof($hander)
    while ($tmpindex < $endnum)
    {
     $tmp = fgets($hander , $this->length);
     if(empty($tmp)) continue;
     if($tmpindex < $endnum  && $tmpindex >=$startnum)
     {
      $tmp = self::_parsedata($tmp);    
      if(empty($tmp)) continue;
      //去掉jsondata
       if(!$jsonflag) { unset($tmp[2]);  $condition= array();  }
      //条件过滤
      if($condition && is_array($condition) )
      {
       foreach ($condition as $key=>$val){
        if(in_array($tmp['data'][$key], $condition[$key]))
         unset($key , $val); 
         $data['count']++;
         $data['data'][$index] = $tmp;
         $index++;
        }
      }
      else{    
       $data['data'][$index] = $tmp;
       $index++;
       $data['count']++;
      }
     }
     if($tmpindex >= $endnum) break;
     $tmpindex++;
     unset($tmp);
    }
    fseek($hander ,  seek_end);
   }

   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';

   return $data;
  }
  

  /**
   * todo:命令行下获取文件总记录数*
   * @param string $date
   * @param int $appid
   * @param string $op
   * @return array
   */
  public  function total_linefile($date, $appid, $op)
  {
   $this->_openfile($date, $appid, $op);
   $file = escapeshellarg($this->_getfile($date, $appid, $op)); // 对命令行参数进行安全转义
   $line = `wc -l  $file`;
   if(preg_match("/(\d{1,})/", $line , $ret)){
    $data['count']=$ret[1];
   }else{
    $data['count']=0;
   }
   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';
   return $data;
  }

  
  /**
   * todo:统计{$data}.{$op}.log记录数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param array $condition
   * @return array
   */
  public function countfile($date , $appid , $op ,$condition=array())
  {
   $data['count'] = 0;
   $hander = self::_openfile($date , $appid , $op );  
   if($hander)
   {
    while (!feof($hander))
    {
     $tmp = fgets($hander , $this->length);
     $tmp = self::_parsedata($tmp);
     if(empty($tmp)) continue;
     if($condition && is_array($condition) )
     {
      foreach ($condition as $key=>$val){
       if(isset($tmp['data'][$key]) && $tmp['data'][$key] && in_array($tmp['data'][$key], $condition[$key])){
        unset($key , $val); 
        $data['count']++;
       } 
      }
     }
     else
      $data['count']++;      
     unset($tmp);
    }
    fseek($hander , seek_end);
   }
   $data['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $data['memory'] = doubleval((doubleval(memory_get_usage(true)) -  doubleval(self::$startmemory))/1024/1024) . ' m';

   return $data;
  }  

  

  /**
   * todo:统计用户数
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param bool $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * * @param bool $jsonflag：默认为true， 则保留jsondata字段；设为false，则去掉jsondata字段
   * @param array $condition
   * @return : array：形如（{"mid":[],"count":2181,"difftime":0.0397667884827,"memory":"3.75 m"}）
   */
  public function countfilemid($date , $appid , $op  ,  $midflag=false , $jsonflag=false,  $condition=array())
  {
   $count = self::total_linefile($date , $appid , $op );
   $index = ceil($count['count'] / $this->limit);
   $result = array('mid'=>array() , 'count'=>0 , 'difftime'=>0 , 'memory'=>0);

   for ($i=0 ; $i<$index ; $i++)
   {
    $startnum = $this->limit*$i;
    $endnum = $this->limit*($i+1);
    $data = self::readfile($date , $appid , $op  ,  $startnum , $endnum , $jsonflag);
    if($data['data'] && is_array($data['data']))
    {
     foreach ($data['data'] as $arr)
     {
      if($condition && is_array($condition)){
       foreach ($condition as $key=>$val){ 
        if(isset($arr['data'][$key])  && (in_array($arr['data'][$key] , $condition[$key]) || empty($condition[$key]))){
         if(!isset($result['mid'][$arr['mid']]))  { $result['mid'][$arr["mid"]] =1; $result['count']++; }
        }
       }
      }
      else {
       if(!isset($result['mid'][$arr['mid']]))  { $result['mid'][$arr["mid"]] =1; $result['count']++;   }
      }
     }
    }
    unset($data['data'] , $data);
   }
   unset($index , $count , $condition , $data  , $arr);
   self::closefile($date , $appid , $op);

    $result['mid'] = array_keys($result['mid']);
    //$result['count'] = count($result['mid']);   
    if(empty($midflag)) unset($result['mid']); 

   $result['difftime'] = doubleval(microtime(true)) - doubleval($this->starttime);
   $result['memory'] = (memory_get_usage(true) - self::$startmemory)/1024/1024 . ' m';
   return $result;
  }

  
  /**
   * todo:跨时间段 统计参加$op用户数据
   * @param string $date
   * @param int $appid
   * @param string $op
   * @param number $day
   * @param bool  $midflag ：默认为false 则 mid返回空数组；如设为true，则mid数组不为空
   * @return array 形如（"20140326":{"mid":[],"count":4571,"difftime":0.0806441307068,"memory":"3.75 m"},
  *          "20140325":{"mid":[],"count":2181,"difftime":0.0397667884827,"memory":"3.75 m"}）
   */
  public function getreturnuser($date , $appid , $op , $day=1 , $midflag=false)
  {
   $date_i=0;
   for ($i =0; $i<$day ; $i++){
    $date_i = date('ymd' , strtotime($date)-$i*86400);
    $result[$date_i] = self::countfilemid($date_i , $appid , $op , $midflag); 
   }
   unset($date , $date_i , $appid , $op  , $day);

   return $result;
  }
 }
?>