欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

php限制curl并发数

程序员文章站 2022-05-05 20:39:09
...
class MultiCurl
{
    private $allToDo;
    private $multiHandle;
    private $maxConcurrent = 2;
    private $currentIndex  = 0;
    private $info          = array();
    private $options       = array(CURLOPT_RETURNTRANSFER => true,
                                   CURLOPT_FOLLOWLOCATION => true,
                                   CURLOPT_MAXREDIRS      => 3,
                                   CURLOPT_TIMEOUT        => 3);

    public function __construct($todo, $concurrent)
    {
        $this->allToDo = $todo;
        $this->maxConcurrent = $concurrent;
        $this->multiHandle = curl_multi_init();
    }
	
    private function _addHandles($num)
    {
		while ($num-- > 0) {
			$handle = curl_init($this->allToDo[$this->currentIndex]);
            curl_setopt_array($handle, $this->options);
            curl_multi_add_handle($this->multiHandle, $handle);
            $this->info[$this->currentIndex] = $handle;
            $this->currentIndex++;
        }
    }        

    private function _moreToDo()
    {
		return count($this->allToDo) - $this->currentIndex;
    }

	public function process()
    {
        do{
			$running = null;
			$concurrent=min($this->maxConcurrent, $this->_moreToDo());
			$this->_addHandles($concurrent);
			// Wait for activity on any curl_multi connection when curl_multi_select (libcurl) fails to correctly block.
            // https://bugs.php.net/bug.php?id=63411
			do {
				$mrc = curl_multi_exec($this->multiHandle, $running); 
			} while ($mrc == CURLM_CALL_MULTI_PERFORM);
			
			while ($running && $mrc == CURLM_OK) {
				 if (curl_multi_select($this->multiHandle) == -1) {
					 usleep(100);
				 }
				 do {
					$mrc = curl_multi_exec($this->multiHandle, $running);
				 } while ($mrc == CURLM_CALL_MULTI_PERFORM);
			}
			
		}while($this->_moreTodo());
		
		foreach($this->info as $info){
			$cur_info=curl_getinfo($info);
			
			$content = curl_multi_getcontent($info);
			echo $cur_info['url'] . ' - ' . strlen($content) . ' bytes<br />';
		
			curl_multi_remove_handle($this->multiHandle, $info);
			curl_close($info);
		}
		curl_multi_close($this->multiHandle);
		
		//$this->clear();
		
        return $this;
    } 
	
	public function clear(){
		$m=memory_get_usage(); //获取当前占用内存  
		echo "memory_usage:",$m,"<br>";  
		$this->multiHandle=null;
		$this->info=null;
		$m=memory_get_usage(); //获取当前占用内存  
		echo "after memory_usage:",$m,"<br>";  
	}
}

function microtime_float()
{
    list($usec, $sec) = explode(" ", microtime());
    return ((float)$usec + (float)$sec);
}

$sites = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
];
$concurrent = 3;   // Any number.
$time_start=microtime_float();
$mc = new MultiCurl($sites, $concurrent);
$mc->process();
$time_end=microtime_float();
echo $time_end - $time_start;

1.php

<?php
sleep(2);

以下为使用guzzle:

<?php
use GuzzleHttp\Client;
use GuzzleHttp\Pool;
use GuzzleHttp\Promise;

require_once 'vendor/autoload.php';


$client=new Client([
	'timeout'=>4
]);

function microtime_float()
{
    list($usec, $sec) = explode(" ", microtime());
    return ((float)$usec + (float)$sec);
}
$urls = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
];
$time_start=microtime_float();

$requests = function () use ($client,$urls) {
    foreach ($urls as $uri) {
        yield function() use ($client, $uri) {
            return $client->getAsync($uri);
        };
    }
};
$pool = new Pool($client, $requests(),[
	'concurrency' => 3,//default 25
    'fulfilled' => function ($response, $index) {
		// this is delivered each successful response
		echo $response->getStatusCode().'<br>';
	},
    'rejected' => function ($reason, $index) {
        // this is delivered each failed request
		echo $reason."<br>";
    },
]);
$promise = $pool->promise();
$promise->wait();

$time_end=microtime_float();
echo $time_end - $time_start;

相关:

python2:grequests

# coding=utf-8
import grequests
import time

start =time.clock()
urls = [
    'http://localhost:88/2018/',
    'http://localhost:88/2018/1.php?v=1',
    'http://localhost:88/2018/1.php?v=2',
    'http://localhost:88/2018/1.php?v=3',
    'http://localhost:88/2018/1.php?v=4',
    'http://localhost:88/2018/1.php?v=5',
    'http://localhost:88/2018/1.php?v=6',
    'http://localhost:88/2018/1.php?v=7',
    'http://localhost:88/2018/1.php?v=8',
    'http://localhost:88/2018/1.php?v=9',
    'http://localhost:88/2018/1.php?v=10',
]
rs = (grequests.get(u,timeout=10) for u in urls)
response=grequests.map(rs)
end = time.clock()
print('Running time: %s Seconds'%(end-start))
print(u'请求完成..')
response0 = response[0]
#print response0.text

python2:requests+gevent

# coding=utf-8
import requests
import gevent
from gevent import monkey
monkey.patch_socket()

urls = ["http://www.baidu.com/"] *10
def get_content(url):
    data = requests.get(url,timeout=10)
    return data
	
#response = requests.get('http://baidu.com',timeout=10)
#print('请求完成..')

jobs = [gevent.spawn(get_content, url) for url in urls]
gevent.joinall(jobs)
print('请求完成..')

python3:requests-futures

import requests
import concurrent.futures

def get_urls():
    return ["url1","url2"]

def load_url(url, timeout):
    return requests.get(url, timeout = timeout)

with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:

    future_to_url = {executor.submit(load_url, url, 10): url for url in     get_urls()}
    for future in concurrent.futures.as_completed(future_to_url):
        url = future_to_url[future]
        try:
            data = future.result()
        except Exception as exc:
            resp_err = resp_err + 1
        else:
            resp_ok = resp_ok + 1

 

转载于:https://my.oschina.net/zengde/blog/1808894