采集远程网址数据

php代码

<?php
/**
 * 采集远程网址数据
 */
class Get_Remote_data
{
	/**
	 * 从远程网址采集数据
	 * @param string $uri 远程地址
	 * @param string $rand_ip 随机ip
	 * @param string $user_agent 模拟的user_agent
	 * @param string $referer_uri 模拟的来路的地址
	 * @param array $post_data 需要post的数据，默认为空数组
	 * @param string $method 选择数据提交的数据，默认get
	 * @return mixed
	 */
	public function get_data_from_remote($uri, $rand_ip, $user_agent, $referer_uri='', $post_data=array(), $method="get")
	{
		$ch = curl_init();
	
		curl_setopt($ch, CURLOPT_URL, $uri);
		curl_setopt($ch, CURLOPT_TIMEOUT, 10);
		curl_setopt($ch, CURLOPT_HTTPHEADER, array('X-FORWARDED-FOR:'.$rand_ip, 'CLIENT-IP:'.$rand_ip));
		//追踪返回302状态码，继续抓取
		// curl_setopt($ch, CURLOPT_HEADER, true);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
		//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
	
		if($method == 'post') {
			curl_setopt($ch, CURLOPT_POST, true);
			curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data);
		}
	
		curl_setopt($ch, CURLOPT_NOBODY, false);
	
		if($method == 'get') {
			curl_setopt($ch, CURLOPT_REFERER, $referer_uri);//模拟来路
		}
	
		curl_setopt($ch, CURLOPT_USERAGENT, $user_agent);
	
		$data = curl_exec($ch); # 获取数据
		$http_code = curl_getinfo($ch,CURLINFO_HTTP_CODE); # 获取http状态码
	
		curl_close($ch);
	
		return array(
				'http_status' => $http_code,
				'data' => $data
		);
	}
	   
    /**
     * 生成模拟来路地址
     * @return string
     */
    private function get_refer_uri()
    {
    	$refer[0] = 'http://www.baidu.com';
    	$refer[1] = 'https://www.google.com';
    
    	$i = time() % count($refer);
    	
    	return $refer[$i];
    }
    
    /**
     * 生成agent
     * @return string
     */
    private function get_user_agent()
    {
    	// windows + chrome
    	$str[0] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36
                    (KHTML, like Gecko) Chrome/29.0.1547.66 Safari/537.36";
    	
    	// windows + ie
    	$str[1] = "Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; MASPJS; rv:11.0) like Gecko";
    
    	$i = time() % count($str);
    	
    	return $str[$i];
    }
    
    /**
     * 生成随机ip
     * @return string
     */
    private function get_rand_ip()
    {
    	$arr_1 = array("218","218","66","66","218","218","60","60","202","204",
    			"66","66","66","59","61","60","222","221","66","59","60",
    			"60","66","218","218","62","63","64","66","66","122","211");
    
    	$randarr= mt_rand(0,count($arr_1));
    	$ip1id = $arr_1[$randarr];
    
    	$ip2id=   round(rand(600000,   2550000)   /   10000);
    	$ip3id=   round(rand(600000,   2550000)   /   10000);
    	$ip4id=   round(rand(600000,   2550000)   /   10000);
    
    	return   $ip1id . "." . $ip2id . "." . $ip3id . "." . $ip4id;
    }
    
}