skymvc网站测试之页面测试,爬行所有页面
程序员文章站
2022-05-17 14:24:18
...
跳至
[1]
[全屏预览]
query("show tables"); $data=M("article")->fetch_array(PDO::FETCH_NUM); foreach($data as $k=>$t){ $tables[]=str_replace(TABLE_PRE,"",$t[0]); } return $tables; } public function onSetAction($a=array()){ if(!empty($a)){ $this->action=array_merge($this->action,$a); } } public function onSetPara($a=array()){ if(!empty($a)){ $this->para=array_merge($this->para,$a); } } public function getUrls(){ $tables=$this->getTables(); $para=""; foreach($this->para as $q){ $para.="&$q=".$this->id; } $para.="&token=".$this->token."&per_page=".$this->per_page; foreach($tables as $m){ $uk=0; foreach($this->action as $k=>$a){ $urls[]=$this->site."m=$m&a=$a".$para; if($k%$this->maxThreads==($this->maxThreads-1)){ $uk++; } } } return $urls; } public function onStart(){ set_time_limit(0); ob_implicit_flush(true); $urls=$this->getUrls(); $uk=0; foreach($urls as $k=>$url){ $purls[$uk][]=$url; if($k%$this->maxThreads==($this->maxThreads-1)){ $uk++; } } echo "开始
"; foreach($purls as $k=>$urls){ echo "第".$k."部分
"; flush(); @ob_flush(); $this->loadClass("spider"); $this->spider->start($urls,function($data){ echo $data['url']."
"; flush(); @ob_flush(); }); } echo "测试结束"; } } ?>
上一篇: 音乐采集程序
下一篇: 网页文字简繁转换函数