使用 PHP 和 cURL 获取 URLs

http://css.dzone.com/articles/retrieving-urls-parallel-curl

1. class Footo_Content_Retrieve_HTTP_CURLParallel
2. {
3. /**
4. * Fetch a collection of URLs in parallell using cURL. The results are
5. * returned as an associative array, with the URLs as the key and the
6. * content of the URLs as the value.
7. *
8. * @param array $addresses An array of URLs to fetch.
9. * @return array The content of each URL that we've been asked to fetch.
10. **/
11. public function retrieve($addresses)
12. {
13. $multiHandle = curl_multi_init();
14. $handles = array();
15. $results = array();
17. foreach($addresses as $url)
18. {
19. $handle = curl_init($url);
20. $handles[$url] = $handle;
22. curl_setopt_array($handle, array(
23. CURLOPT_HEADER => false,
24. CURLOPT_RETURNTRANSFER => true,
25. ));
27. curl_multi_add_handle($multiHandle, $handle);
28. }
30. //execute the handles
31. $result = CURLM_CALL_MULTI_PERFORM;
32. $running = false;
34. // set up and make any requests..
35. while ($result == CURLM_CALL_MULTI_PERFORM)
36. {
37. $result = curl_multi_exec($multiHandle, $running);
38. }
40. // wait until data arrives on all sockets
41. while($running && ($result == CURLM_OK))
42. {
43. if (curl_multi_select($multiHandle) > -1)
44. {
45. $result = CURLM_CALL_MULTI_PERFORM;
47. // while we need to process sockets
48. while ($result == CURLM_CALL_MULTI_PERFORM)
49. {
50. $result = curl_multi_exec($multiHandle, $running);
51. }
52. }
53. }
55. // clean up
56. foreach($handles as $url => $handle)
57. {
58. $results[$url] = curl_multi_getcontent($handle);
60. curl_multi_remove_handle($multiHandle, $handle);
61. curl_close($handle);
62. }
64. curl_multi_close($multiHandle);
66. return $results;
67. }
68. }

复制代码