php写的关于静态页面的蜘蛛爬行记录的代码

3. $useragent = addslashes(strtolower($_SERVER['HTTP_USER_AGENT']));

4. if (strpos($useragent, 'googlebot')!== false){$bot = 'Google';}
5. elseif (strpos($useragent,'mediapartners-google') !== false){$bot = 'Google Adsense';}
6. elseif (strpos($useragent,'baiduspider') !== false){$bot = 'Baidu';}
7. elseif (strpos($useragent,'sogou spider') !== false){$bot = 'Sogou';}
8. elseif (strpos($useragent,'sogou web') !== false){$bot = 'Sogou web';}
9. elseif (strpos($useragent,'sosospider') !== false){$bot = 'SOSO';}
10. elseif (strpos($useragent,'yahoo') !== false){$bot = 'Yahoo';}
11. elseif (strpos($useragent,'msn') !== false){$bot = 'MSN';}
12. elseif (strpos($useragent,'msnbot') !== false){$bot = 'msnbot';}
13. elseif (strpos($useragent,'sohu') !== false){$bot = 'Sohu';}
14. elseif (strpos($useragent,'yodaoBot') !== false){$bot = 'Yodao';}
15. elseif (strpos($useragent,'twiceler') !== false){$bot = 'Twiceler';}
16. elseif (strpos($useragent,'ia_archiver') !== false){$bot = 'Alexa_';}
17. elseif (strpos($useragent,'iaarchiver') !== false){$bot = 'Alexa';}
18. elseif (strpos($useragent,'slurp') !== false){$bot = '雅虎';}
19. elseif (strpos($useragent,'bot') !== false){$bot = '其它蜘蛛';}
20. if(isset($bot)){
21. $fp = @fopen('bot.txt','a');
22. fwrite($fp,date('Y-m-d H:i:s')."\t".$_SERVER["REMOTE_ADDR"]."\t".$bot."\t".'http://'.$_SERVER['SERVER_NAME'].$_SERVER["HTTP_X_REWRITE_URL"]."\r\n");
23. fclose($fp);
24. }
25. $file=".".$_SERVER[HTTP_X_REWRITE_URL];
26. $f_head=substr($file,-5);
27. if($f_head==".html")
28. {
29. if(file_exists($file))
30. {
31. echo file_get_contents($file);
32. }else
33. {
35. header('HTTP/1.1 404 Not Found');
36. header("status: 404 Not Found");

37. echo "该页面无法找到";

38. }
39. }
40. else
41. {
42. header('HTTP/1.1 404 Not Found');
43. header("status: 404 Not Found");
44. echo "该页面无法找到";
45. }
46. ?>

复制代码

伪静态文件内容：

3. [ISAPI_Rewrite]

4. # 3600 = 1 hour

5. CacheClockRate 3600
7. RepeatLimit 32
9. # Protect httpd.ini and httpd.parse.errors files
10. # from accessing through HTTP
11. RewriteRule /index.html /index.php
12. RewriteRule ^/article/(.*) /bot.php [L]
13. RewriteRule ^/list/(.*) /bot.php [L]

复制代码