利用curl，正则表达式做的一个php蜘蛛抓取器

凤网fcms内容管理系统
get.php 抓取框架，对网页内容的分析处理并进行相关替换
std.php 通用正则
news_67_com.php 对http://news.67.com 的抓取分析器
先抓列表，再抓内容页。
还欠缺监控，统计，错误处理功能。个人觉得还是比较好玩。

1. include_once dirname(__FILE__) . '/std.php';
3. $site = array(
4. 'aname' => '中国娱乐网',
5. 'domain' => 'news.67.com',
6. 'dirname' => '目录名称，用于匹配基于目录不同的正文',
7. 'gettype' => 'default',
8. //获取主文件
9. 'creg' => '/(?si)(.*?)\/',
10. 'code' => 'utf-8',
11. 'sub' => '获取子目录正则',
12. 'content' => 'tag1',
13. 'img_upload'=> array('tag1' => ''),
14. //下一页
15. 'reg_next' => '/(?is)下一页\>\>/',
16. 'key0' => '/(?is)/',
17. 'key0_ap' => array(array(',', '|'), ' '),
18. 'tag0' => '/(?is)

    ([^]*?)/',

19. 'tag0_arp' => array(
20. array(
21. '/(?is)\(组图\)/',
22. '/(?is)\(图\)/',
23. '/(?is)\(图\.\./',
24. '/(?is)\(组图\.\./',
25. '/(?is)\./',
26. '/(?is)(《|》)/',
27. ),
28. array(
29. '', '', '', '', '', '',
30. )
31. ),
32. 'tag1' => '/(?is)
    (.*?)/',
33. 'tag1_brp' => array(
34. array(
35. '/(?is)（.*?）/',
36. '/(?is)\(.*?\)/',
37. '/(?is)\s*

    .*?(.*?)\s*/',

38. '/(?is)\s*

    \s*/',

39. '/(?is)\s*

    \s*/',

40. '/(?is)　/',
41. '/(?is)
    /',
42. '/(?is)\s*

    \s*/',

43. '/(?is)\s*

    \s*/',

44. '/(?is)\s*\s*/',
    '/(?is)\s*\s*/',
    '/(?is)\s*

    \s*/',

    ),
    array(
    '', '', '

    

    ', '

    ', '

    ', '', '', '

    ', '

    ', '

    ', '

    ', '

    '

    ),
    ),
    'tag1_arp' => array(
    array(
    '/(?is)

    /',

    '/(?is)/'
    ),
    array(
    '', ''
    ),
    ),
    'strip' => array('tag1' => ''),
    'tag2' => '/(?is)

    导读：\s*(.*?)\s*/',
    'tag2_arp' => array(
    array(
    '/(?is)　/'
    ),
    array(
    ''
    ),
    ),
    'tag3' => '/(?is)(中国娱乐网)/',
    'tag4' => '/(?is)

    日期：(\d+-\d+-\d+ \d+:\d+:\d+).*?/',
    );
    $map = array(
    'tag' => 'key0',
    'title' => 'tag0',
    'content' => 'tag1',
    'summary' => 'tag2',
    'source' => 'tag3',
    'pub_date' => 'tag4',
    );
    $site_list = array(
    'aname' => '中国娱乐网',
    'domain' => 'www.67.com',
    'gettype' => 'default',
    'creg' => '/(?si)

    (.*?)

    /',
    'code' => 'gbk',
    'reg_next' => '/(?si)
    下一页/',
    //链接
    'tag0' => '/(?is)

    .*?[^]*?.*?/',
    //标题
    'tag1' => '/(?is)

    .*?([^]*?).*?/',
    'tag1_arp' => array(
    array(
    '/(?is)\(组图\)/',
    '/(?is)\(图\)/',
    '/(?is)\(图\.\./',
    '/(?is)\(组图\.\./',
    '/(?is)\./',
    '/(?is)(《|》)/',
    ),
    array(
    '', '', '', '', '', '',
    )
    ),
    );
    $list_map = array(
    'url' => 'tag0',
    'title' => 'tag1',
    );
    $site_list_sub = array();

复制代码

1. global $std;
2. $std = array(
3. 'url' => '[0-9a-zA-Z\.\:\-\/%_#;&]+',
4. 'img' => '/(?is)/',
5. );

复制代码

1. /**
2. * test.php
3. *
4. * @author xzfred
5. * @copyright 2009 fengone.com
6. * @created 2010-12-07 .
7. * @version $Id: php.php 3 2008-10-10 07:49:21Z fred $
8. * SVNPath $HeadURL: http://192.168.0.16/svn/vim/skeletons/php.php $
9. */
10. /*
11. include_once "std.php";
12. include_once "lady_163_com.php";
13. */
14. include_once $GLOBALS['g_dir_core'] . "get.php";
16. //================================================================================
17. include_once DIR_HOST_TAG . '/tuku_ent_china_com.php';
18. $obj = new FcHtmlParse($site);
19. $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/2011-08-23/181703.xml"));
20. echo "\n\n\n ===================\n";
21. echo $c['field']['tag1'][0];
22. echo "\n\n\n ===================\n";
23. var_dump($c);
25. exit();
26. //列表测试
27. $obj = new FcHtmlParse($site_list);
28. $c = $obj->parse(file_get_contents("http://tuku.ent.china.com/fun/html/3569_1.html"));
29. var_dump($c);
30. exit();
31. /*
33. $obj = new FcHtmlGet($site);
34. $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
35. var_dump($c);
37. $obj = new FcHtmlGet($site);
38. $c = $obj->getPage('http://star.pclady.com.cn/entertainment/ss/1106/703240.html');
39. var_dump($c);
41. $obj = new FcHtmlParse($site);
42. $img_obj = new FcHtmlImgUpload($site);
44. $data = file_get_contents("e:/b.html");
45. $c = $obj->parse($data);
46. $ic = $img_obj->upload($c['tag']['tag1'][0]);
47. var_dump($ic);
49. $data = file_get_contents("e:/a.html");
50. $c = $obj->parse($data);
51. $ic = $img_obj->upload($c['tag']['tag1'][0]);
52. var_dump($ic);
53. */
55. //var_dump($c['tag']['tag1']);

复制代码

1. include_once $GLOBALS['g_dir_core'] . 'host/std.php';
3. $site = array(
4. 'aname' => '中华网娱乐图库',
5. 'domain' => 'tuku.ent.china.com',
6. 'dirname' => '目录名称，用于匹配基于目录不同的正文',
7. 'gettype' => 'default',
8. //获取主文件
9. 'creg' => '/(?si)(.*?)/',
10. 'code' => 'utf-8',
11. 'sub' => '获取子目录正则',
12. 'content' => 'tag1',
13. 'img_upload'=> array('tag1' => ''),
14. //下一页
15. 'reg_next' => '/(?is)下一页/',
16. 'key0' => '/(?is)/',
17. 'key0_ap' => array(array(',', '|'), ' '),
18. 'tag0' => '/(?is)title="([^"]*?)"/',
19. 'tag0_arp' => array(
20. array(
21. '/(?is)\(图\)/',
22. '/(?is)\"/',
23. '/(?is)独家：/',
24. '/(?is)独家:/',
25. '/(?is)(《|》)/',
26. ),
27. array(
28. '', '', '', '', '',
29. )
30. ),
31. 'tag1' => '/(?is)(.*?)/',
32. 'tag1_brp' => array(
33. array(
34. '/(?is)\s*\s*/'
35. ),
36. array(
37. '

    

38. $1

39. $3

    '
40. )
41. ),
42. 'tag1_arp' => array(
43. array(
44. '/(?is)

    ([^]*?)/',

45. '/(?is)\<br\/\>/',
46. ),
47. array(
48. '

    $1

    ',
49. '',
50. )
51. ),
52. 'strip' => array('tag1' => ''),
53. //网名
54. 'tag3' => '/(?is)([^]*?)/',
55. 'tag4' => '/(?is)(中华网)/'
56. );
58. $map = array(
59. 'tag' => 'key0',
60. 'title' => 'tag0',
61. 'content' => 'tag1',
62. 'author' => 'tag3',
63. 'source' => 'tag4'
64. );
66. $site_list = array(
67. 'aname' => '中华网娱乐图库',
68. 'domain' => 'tuku.ent.china.com',
69. 'gettype' => 'default',
70. 'creg' => '/(?si)
    (.*?)

    /',
71. 'code' => 'utf-8',
72. 'reg_next' => '/(?si)
73. 下一页/',
74. //链接
75. 'tag0' => '/(?is)
    .*?[^]*?/',
76. 'tag0_brp' => array(
77. array(
78. '/(?is)\.htm/',
79. ),
80. array(
81. '.xml'
82. )
83. ),
84. //标题
85. 'tag1' => '/(?is)
    .*?([^]*?)/',
86. 'tag1_arp' => array(
87. array(
88. '/(?is)\(图\)/',
89. '/(?is)\"/',
90. '/(?is)独家：/',
91. '/(?is)独家:/',
92. '/(?is)(《|》)/',
93. ),
94. array(
95. '', '', '', '', '',
96. )
97. ),
98. );
100. $list_map = array(
101. 'url' => 'tag0',
102. 'title' => 'tag1',
103. );
105. $site_list_sub = array();

复制代码