php限制curl并发数

2019独角兽企业重金招聘Python工程师标准>>>

class MultiCurl
{
    private $allToDo;
    private $multiHandle;
    private $maxConcurrent = 2;
    private $currentIndex  = 0;
    private $info          = array();
    private $options       = array(CURLOPT_RETURNTRANSFER => true,
                                   CURLOPT_FOLLOWLOCATION => true,
                                   CURLOPT_MAXREDIRS      => 3,
                                   CURLOPT_TIMEOUT        => 3);

    public function __construct($todo, $concurrent)
    {
        $this->allToDo = $todo;
        $this->maxConcurrent = $concurrent;
        $this->multiHandle = curl_multi_init();
    }
	
    private function _addHandles($num)
    {
		while ($num-- > 0) {
			$handle = curl_init($this->allToDo[$this->currentIndex]);
            curl_setopt_array($handle, $this->options);
            curl_multi_add_handle($this->multiHandle, $handle);
            $this->info[$this->currentIndex] = $handle;
            $this->currentIndex++;
        }
    }        

    private function _moreToDo()
    {
		return count($this->allToDo) - $this->currentIndex;
    }

	public function process()
    {
        do{
			$running = null;
			$concurrent=min($this->maxConcurrent, $this->_moreToDo());
			$this->_addHandles($concurrent);
			// Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.
            // https://bugs.php.net/bug.php?id=63411
			do {
				$mrc = curl_multi_exec($this->multiHandle, $running); 
			} while ($mrc == CURLM_CALL_MULTI_PERFORM);
			
			while ($running && $mrc == CURLM_OK) {
				 if (curl_multi_select($this->multiHandle) == -1) {
					 usleep(100);
				 }
				 do {
					$mrc = curl_multi_exec($this->multiHandle, $running);
				 } while ($mrc == CURLM_CALL_MULTI_PERFORM);
			}
			
		}while($this->_moreTodo());
		
		foreach($this->info as $info){
			$cur_info=curl_getinfo($info);
			
			$content = curl_multi_getcontent($info);
			echo $cur_info['url'] . ' - ' . strlen($content) . ' bytes<br />';
		
			curl_multi_remove_handle($this->multiHandle, $info);
			curl_close($info);
		}
		curl_multi_close($this->multiHandle);
		
		//$this->clear();
		
        return $this;
    } 
	
	public function clear(){
		$m=memory_get_usage(); //获取当前占用内存  
		echo "memory_usage:",$m,"<br>";  
		$this->multiHandle=null;
		$this->info=null;
		$m=memory_get_usage(); //获取当前占用内存  
		echo "after memory_usage:",$m,"<br>";  
	}
}

function microtime_float()
{
    list($usec, $sec) = explode(" ", microtime());
    return ((float)$usec + (float)$sec);
}

$sites = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
];
$concurrent = 3;   // Any number.
$time_start=microtime_float();
$mc = new MultiCurl($sites, $concurrent);
$mc->process();
$time_end=microtime_float();
echo $time_end - $time_start;

1.php

<?php
sleep(2);

以下为使用guzzle：

<?php
use GuzzleHttp\Client;
use GuzzleHttp\Pool;
use GuzzleHttp\Promise;

require_once 'vendor/autoload.php';


$client=new Client([
	'timeout'=>4
]);

function microtime_float()
{
    list($usec, $sec) = explode(" ", microtime());
    return ((float)$usec + (float)$sec);
}
$urls = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
];
$time_start=microtime_float();

$requests = function () use ($client,$urls) {
    foreach ($urls as $uri) {
        yield function() use ($client, $uri) {
            return $client->getAsync($uri);
        };
    }
};
$pool = new Pool($client, $requests(),[
	'concurrency' => 3,//default 25
    'fulfilled' => function ($response, $index) {
		// this is delivered each successful response
		echo $response->getStatusCode().'<br>';
	},
    'rejected' => function ($reason, $index) {
        // this is delivered each failed request
		echo $reason."<br>";
    },
]);
$promise = $pool->promise();
$promise->wait();

$time_end=microtime_float();
echo $time_end - $time_start;

相关：

python2:grequests

# coding=utf-8
import grequests
import time

start =time.clock()
urls = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
]
rs = (grequests.get(u,timeout=10) for u in urls)
response=grequests.map(rs)
end = time.clock()
print('Running time: %s Seconds'%(end-start))
print(u'请求完成..')
response0 = response[0]
#print response0.text

python2:requests+gevent

# coding=utf-8
import requests
import gevent
from gevent import monkey
monkey.patch_socket()

urls = ["http://www.baidu.com/"] *10
def get_content(url):
    data = requests.get(url,timeout=10)
    return data
	
#response = requests.get('http://baidu.com',timeout=10)
#print('请求完成..')

jobs = [gevent.spawn(get_content, url) for url in urls]
gevent.joinall(jobs)
print('请求完成..')

python3:requests-futures

import requests
import concurrent.futures

def get_urls():
    return ["url1","url2"]

def load_url(url, timeout):
    return requests.get(url, timeout = timeout)

with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:

    future_to_url = {executor.submit(load_url, url, 10): url for url in     get_urls()}
    for future in concurrent.futures.as_completed(future_to_url):
        url = future_to_url[future]
        try:
            data = future.result()
        except Exception as exc:
            resp_err = resp_err + 1
        else:
            resp_ok = resp_ok + 1

 

转载于:https://my.oschina.net/zengde/blog/1808894