在Python中使用CasperJS获取JS渲染生成的HTML内容的教程
程序员文章站
2022-06-08 12:49:39
...
文章摘要：其实这里casperjs与python没有直接关系,主要依赖casperjs调用phantomjs webkit获取html文件内容。长期以来，爬虫抓取 客户端javascript渲染生成的html页面 都极为 困难, Java里面有 HtmlUnit, 而Python里，我们可以使用独立的跨平台的CasperJS。
创建site.js(接口文件，输入:url，输出:html file)
//USAGE: E:\toolkit\n1k0-casperjs-e3a77d0\bin>python casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile='temp.html'
var fs = require('fs');
var casper = require('casper').create({
pageSettings: {
loadImages: false,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36 LBBROWSER'
},
logLevel: "debug",//日志等级
verbose: true // 记录日志到控制台
});
var url = casper.cli.raw.get('url');
var outputfile = casper.cli.raw.get('outputfile');
//请求页面
casper.start(url, function () {
fs.write(outputfile, this.getHTML(), 'w');
});
casper.run();
python 代码, checkout_proxy.py
import json
import sys
#import requests
#import requests.utils, pickle
from bs4 import BeautifulSoup
import os.path,os
import threading
#from multiprocessing import Process, Manager
from datetime import datetime
import traceback
import logging
import re,random
import subprocess
import shutil
import platform
output_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'proxy.txt')
global_log = 'http_proxy' + datetime.now().strftime('%Y-%m-%d') + '.log'
if not os.path.exists(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs')):
os.mkdir(os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs'))
global_log = os.path.join(os.path.dirname(os.path.realpath(__file__)),'logs',global_log)
logging.basicConfig(level=logging.DEBUG,format='[%(asctime)s] [%(levelname)s] [%(module)s] [%(funcName)s] [%(lineno)d] %(message)s',filename=global_log,filemode='a')
log = logging.getLogger(__name__)
#manager = Manager()
#PROXY_LIST = manager.list()
mutex = threading.Lock()
PROXY_LIST = []
def isWindows():
if "Windows" in str(platform.uname()):
return True
else:
return False
def getTagsByAttrs(tagName,pageContent,attrName,attrRegValue):
soup = BeautifulSoup(pageContent)
return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })
def getTagsByAttrsExt(tagName,filename,attrName,attrRegValue):
if os.path.isfile(filename):
f = open(filename,'r')
soup = BeautifulSoup(f)
f.close()
return soup.find_all(tagName, { attrName : re.compile(attrRegValue) })
else:
return None
class Site1Thread(threading.Thread):
def __init__(self,outputFilePath):
threading.Thread.__init__(self)
self.outputFilePath = outputFilePath
self.fileName = str(random.randint(100,1000)) + ".html"
self.setName('Site1Thread')
def run(self):
site1_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),'site.js')
site2_file = os.path.join(self.outputFilePath,'site.js')
if not os.path.isfile(site2_file) and os.path.isfile(site1_file):
shutil.copy(site1_file,site2_file)
#proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
if isWindows():
proc = subprocess.Popen(["cmd","/c", "%s/casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
else:
proc = subprocess.Popen(["bash","-c", "cd %s && ./casperjs site.js --url=http://spys.ru/free-proxy-list/IE/ --outputfile=%s" % (self.outputFilePath,self.fileName) ],stdout=subprocess.PIPE)
out=proc.communicate()[0]
htmlFileName = ''
#因为输出路径在windows不确定，所以这里加了所有可能的路径判断
if os.path.isfile(self.fileName):
htmlFileName = self.fileName
elif os.path.isfile(os.path.join(self.outputFilePath,self.fileName)):
htmlFileName = os.path.join(self.outputFilePath,self.fileName)
elif os.path.isfile(os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)):
htmlFileName = os.path.join(os.path.dirname(os.path.realpath(__file__)),self.fileName)
if (not os.path.isfile(htmlFileName)):
print 'Failed to get html content from http://spys.ru/free-proxy-list/IE/'
print out
sys.exit(3)
mutex.acquire()
PROXYList= getTagsByAttrsExt('font',htmlFileName,'class','spy14$')
for proxy in PROXYList:
tdContent = proxy.renderContents()
lineElems = re.split('[]',tdContent)
if re.compile(r'\d+').search(lineElems[-1]) and re.compile('(\d+\.\d+\.\d+)').search(lineElems[0]):
print lineElems[0],lineElems[-1]
PROXY_LIST.append("%s:%s" % (lineElems[0],lineElems[-1]))
mutex.release()
try:
if os.path.isfile(htmlFileName):
os.remove(htmlFileName)
except:
pass
if __name__ == '__main__':
try:
if(len(sys.argv)) 
声明：本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有涉嫌抄袭侵权的内容，请联系admin@php.cn核实处理。
相关文章
相关视频
专题推荐
-
独孤九贱-php全栈开发教程
全栈 170W+
主讲：Peter-Zhu 轻松幽默、简短易学，非常适合PHP学习入门
-
玉女心经-web前端开发教程
入门 80W+
主讲：灭绝师太 由浅入深、明快简洁，非常适合前端学习入门
-
天龙八部-实战开发教程
实战 120W+
主讲：西门大官人 思路清晰、严谨规范，适合有一定web编程基础学习
上一篇： PHP spl_autoload_register实现自动加载研究
下一篇： PHP中的后期绑定
网友评论
文明上网理性发言，请遵守 新闻评论服务协议
我要评论