欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识！
您现在的位置是： 首页  >  后端开发

自己写的一个php基于phpQuery的通用采集类

程序员文章站 2024-01-01 09:38:40
...
还是小菜，第一次分享代码哈，这是自己以前写的一个php的采集类，自己一直在用，自我感觉很简单很强大，只要懂一点点选择器的知识就可以采集任何页面了，也支持https页面，做简单的采集足够用了。
  2. /**
  3. *通用列表采集类
  4. *版本V1.3
  5. *作者:JAE
  6. *博客:http://blog.jaekj.com
  7. */
  8. require_once '../phpQuery/phpQuery/phpQuery.php';
  9. class QueryList{
  11. private $pageURL;
  12. private $regArr = array();
  13. public $jsonArr = array();
  14. private $regRange;
  15. private $html;
  16. /************************************************
  17. * 参数: 页面地址 选择器数组 块选择器
  18. * 【选择器数组】说明：格式array("名称"=>array("选择器","类型"),.......)
  19. * 【类型】说明：值 "text" ,"html" ,"属性"
  20. *【块选择器】：指 先按照规则 选出 几个大块 ，然后再分别再在块里面 进行相关的选择
  21. *************************************************/
  22. function QueryList($pageURL,$regArr=array(),$regRange='')
  23. {
  24. $this->pageURL = $pageURL;
  26. //为了能获取https://
  27. $ch = curl_init();
  28. curl_setopt($ch, CURLOPT_URL,$this->pageURL);
  29. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
  30. curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
  31. curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
  32. $this->html = curl_exec($ch);
  33. curl_close($ch);
  35. if(!empty($regArr))
  36. {
  38. $this->regArr = $regArr;
  39. $this->regRange = $regRange;
  40. $this->getList();
  41. }
  43. }
  44. function setQuery($regArr,$regRange='')
  45. {
  46. $this->jsonArr=array();
  47. $this->regArr = $regArr;
  48. $this->regRange = $regRange;
  49. $this->getList();
  50. }
  51. private function getList()
  52. {
  54. $hobj = phpQuery::newDocumentHTML($this->html);
  55. if(!empty($this->regRange))
  56. {
  57. $robj = pq($hobj)->find($this->regRange);
  59. $i=0;
  60. foreach($robj as $item)
  61. {
  63. while(list($key,$reg_value)=each($this->regArr))
  64. {
  65. $iobj = pq($item)->find($reg_value[0]);
  67. switch($reg_value[1])
  68. {
  69. case 'text':
  70. $this->jsonArr[$i][$key] = trim(pq($iobj)->text());
  71. break;
  72. case 'html':
  73. $this->jsonArr[$i][$key] = trim(pq($iobj)->html());
  74. break;
  75. default:
  76. $this->jsonArr[$i][$key] = pq($iobj)->attr($reg_value[1]);
  77. break;
  79. }
  80. }
  81. //重置数组指针
  82. reset($this->regArr);
  83. $i++;
  84. }
  85. }
  86. else
  87. {
  88. while(list($key,$reg_value)=each($this->regArr))
  89. {
  90. $lobj = pq($hobj)->find($reg_value[0]);
  93. $i=0;
  94. foreach($lobj as $item)
  95. {
  96. switch($reg_value[1])
  97. {
  98. case 'text':
  99. $this->jsonArr[$i++][$key] = trim(pq($item)->text());
  100. break;
  101. case 'html':
  102. $this->jsonArr[$i++][$key] = trim(pq($item)->html());
  103. break;
  104. default:
  105. $this->jsonArr[$i++][$key] = pq($item)->attr($reg_value[1]);
  106. break;
  108. }
  111. }
  114. }
  115. }
  116. }
  117. function getJSON()
  118. {
  119. return json_encode($this->jsonArr);
  120. }
  122. }
复制代码
  1. require 'Query/QueryList.class.php';
  4. //采集OSC的代码分享列表，标题 链接 作者
  5. $url = "http://www.oschina.net/code/list";
  6. $reg = array("title"=>array(".code_title a:eq(0)","text"),"url"=>array(".code_title a:eq(0)","href"),"author"=>array("img","title"));
  7. $rang = ".code_list li";
  8. $hj = new QueryList($url,$reg,$rang);
  9. $arr = $hj->jsonArr;
  10. print_r($arr);
  11. //如果还想采当前页面右边的 TOP40活跃贡献者 图像，得到JSON数据,可以这样写
  12. $reg = array("portrait"=>array(".hot_top img","src"));
  13. $hj->setQuery($reg);
  14. $json = $hj->getJSON();
  15. echo $json . "
    ";
  17. //采OSC内容页内容
  18. $url = "http://www.oschina.net/code/snippet_186288_23816";
  19. $reg = array("title"=>array(".QTitle h1","text"),"con"=>array(".Content","html"));
  20. $hj = new QueryList($url,$reg);
  21. $arr = $hj->jsonArr;
  22. print_r($arr);
  24. //就举这么多例子吧，是不是用来做采集很方便
复制代码
  2. /**
  3. *自己写的百度和谷歌搜索API
  4. *版本V2.0
  5. *作者:JAE
  6. *博客:http://blog.jaekj.com
  7. **/
  8. require_once 'QueryList_class.php';
  9. class Searcher
  10. {
  11. private $searcher;
  12. private $key;
  13. private $num;
  14. private $page;
  15. private $regArr ;
  16. private $regRange ;
  17. private $regZnum;
  18. public $jsonArr;
  19. //参数 搜索引擎 搜索关键字 返回的结果条数 第几页
  20. function Searcher($searcher,$key,$num,$page)
  21. {
  22. if($searcher=='baidu')
  23. {
  24. $this->regArr = array("title"=>array("h3.t a,#ting_singlesong_box a","text"),"tCon"=>array("div.c-abstract,font:slice(0,2),div#weibo,table tr:eq(0),div.c-abstract-size p:eq(0),div.vd_sitcom_new_tinfo","text"),"url"=>array("h3.t a,#ting_singlesong_box a","href"));
  25. $this->regRange = 'table.result,table.result-op';
  26. $this->regZnum=array("zNum"=>array("span.nums","text"));
  27. }
  28. else if($searcher=='google')
  29. {
  30. $this->regArr = array("title"=>array("h3.r a","text"),"tCon"=>array("span.st","text"),"url"=>array("h3.r a","href"));
  31. $this->regRange = 'li.g';
  32. $this->regZnum=array("zNum"=>array("div#resultStats","text"));
  33. }
  34. $this->searcher = $searcher;
  35. $this->key = $key;
  36. $this->num = $num;
  37. $this->page = $page-1;
  38. $this->getList();
  39. }
  40. private function getList()
  41. {
  42. $s = urlencode($this->key);
  43. $num = $this->num;
  44. $start = $this->num*$this->page;
  45. if($this->searcher=='baidu')
  46. {
  47. $url = "http://www.baidu.com/s?pn=$start&rn=$num&wd=$s";
  48. $reg_znum='/[\d,]+/';
  49. }
  50. else if($this->searcher=='google')
  51. {
  52. $url="https://www.google.com.hk/search?filter=0&lr=&newwindow=1&safe=images&hl=en&as_qdr=all&num=$num&start=$start&q=$s";
  53. $reg_znum='/([\d,]+) result(s)?/';
  54. }
  55. $searcherObj = new QueryList($url,$this->regArr,$this->regRange);
  56. for($i=0;$ijsonArr);$i++)
  57. {
  58. if($this->searcher=='baidu')
  59. {
  60. $searcherObj->jsonArr[$i]['url'] = $this->getBaiduRealURL($searcherObj->jsonArr[$i]['url']);
  61. }
  62. else if($this->searcher=='google')
  63. {
  64. $searcherObj->jsonArr[$i]['url'] = $this->getGoogleRealURL($searcherObj->jsonArr[$i]['url']);
  65. }
  66. }
  67. $this->jsonArr = $searcherObj->jsonArr ;
  69. //获取总共结果条数
  71. $searcherObj->setQuery($this->regZnum);
  72. $zNum = $searcherObj->jsonArr[0]['zNum'];
  73. preg_match($reg_znum,$zNum,$arr)?$zNum=$arr[0]:$zNum=0;
  74. $zNum = (int)str_replace(',','',$zNum);
  75. //计算总页数
  76. $zPage = ceil($zNum/$this->num);
  77. $this->jsonArr=array('num'=>$this->num,'page'=>((int)$this->page+1),'zNum'=>$zNum,'zPage'=>$zPage,"s"=>"$this->key",'other'=>array('author'=>'JAE','QQ'=>'734708094','blog'=>'http://blog.jaekj.com'),'data'=>$this->jsonArr);
  80. }
  81. function getJSON()
  82. {
  83. return json_encode($this->jsonArr);
  84. }
  85. private function getBaiduRealURL($url)
  86. {
  87. //得到百度跳转的真正地址
  88. $header = get_headers($url,1);
  89. if (strpos($header[0],'301') || strpos($header[0],'302'))
  90. {
  91. if(is_array($header['Location']))
  92. {
  93. //return $header['Location'][count($header['Location'])-1];
  94. return $header['Location'][0];
  95. }
  96. else
  97. {
  98. return $header['Location'];
  99. }
  100. }
  101. else
  102. {
  103. return $url;
  104. }
  105. }
  106. private function getGoogleRealURL($url)
  107. {
  108. $reg_url = '/q=(.+)&/U';
  109. return preg_match($reg_url,$url,$arr)?urldecode($arr[1]):$url;
  111. }
  112. }
  113. // $hj = new Searcher('google','oschina',20,2);
  114. // print_r( $hj->jsonArr);
  115. //效果演示地址
  116. //http://blog.jaekj.com//jae/demo/searcher/Searcher_class.php?searcher=baidu&s=jaekj&num=20&page=1
复制代码
相关标签： 自己写的一个php基于phpQuery的通用采集类

上一篇：

下一篇：

推荐阅读