欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页  >  数据库

selenium和casperjs2种数据抓取方式(进来的朋友请留言,共同探讨

程序员文章站 2022-06-15 20:01:29
...

今天用selenium和casperjs2种对https://class.coursera.org/nlp/lecture网站的ppt、pdf、srt、MP4的下载地址进行数据抓取 1、pythonselenium #!/usr/bin/python# -*- coding: utf-8 -*-from selenium import webdriverfrom bs4 import BeautifulSoupimport t

今天用selenium和casperjs2种对https://class.coursera.org/nlp/lecture网站的ppt、pdf、srt、MP4的下载地址进行数据抓取

1、python+selenium

#!/usr/bin/python
# -*- coding: utf-8 -*-

from selenium import webdriver
from bs4 import BeautifulSoup

import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

def catchDate(s):
    """页面数据提取"""
    soup = BeautifulSoup(s)
    z = []
    
    m = soup.findAll("ul",class_="course-item-list-div-list")
    
    for obj in m:
        try:
            print obj.previous_sibling.find('h3').get_text()
            tmp = obj.findAll('li', class_="unviewed")
            for eachli in tmp:
                titleli = eachli.find('a').get_text()
                print '          '+titleli
                allaInEachDiv = eachli.find('div', class_="course-lecture-item-resource").findAll('a')
                for eacha in allaInEachDiv:
                    print '      '+eacha['href']
        except Exception, e:
            continue
        if(tmp != ""):
            z.append(tmp)
    return z

starttime = time.time()
driver = webdriver.PhantomJS(executable_path='C:\phantomjs-1.9.7-windows\phantomjs.exe')
driver.get("https://class.coursera.org/nlp/lecture")
html = driver.page_source
content = catchDate(html)
endtime = time.time()
print endtime - starttime
driver.quit

2、casperjs
var casper = require("casper").create({
	clientScripts: ["jquery-1.7.js"],  
    stepTimeout: 120 * 1000,  
    pageSettings: {  
        loadImages: false  
    },  
    verbose: true,  
    logLevel: "error"  
});  
var numberOfLinks = 0;

var fs = require('fs');
var filename = 'content.txt';
var fullContent = "";
var startTime = new Date(), endTime;
  
casper.start("https://class.coursera.org/nlp/lecture", function() {
    numberOfLinks = this.evaluate(function() {
        return __utils__.findAll('.course-item-list-div-list').length;
    });
    this.echo(numberOfLinks + " items found");
});
getStartTime = function(){
	this.echo(startTime);
	this.then(getcontent);
};
getcontent = function() {
    fullContent = this.evaluate(function() {
        var content = "";
        jQuery('.course-item-list-div-list').each(function() {
			var btitle = $(this).prev().find("h3").text();
			content += btitle + '\r\n';
			$(this).find("li").each(function(){
				var stitle = $(this).find("a").first().text();
				content += stitle + '\r';
				$(this).find("div a").each(function(){
					content += $(this).attr("href")+'\r';
				});
				content += '\r\n';
			});
			content += '\r\n\r\n';
        });
        return content;
    });
	this.then(writefile);
};

writefile = function() {
    this.echo('writing to ' + filename);
    fs.write(filename, fullContent, 'w');
	this.then(getEndTime);
};
getEndTime = function(){
	endTime = new Date();
}
casper.then(getStartTime);
casper.then(function exitSystem() {
	this.echo(new Date() - startTime);
    casper.exit();  
}); 

casper.run();

因为不熟练,感觉写的不太好,求大神对方法进行指导!!!


参考:

https://gist.github.com/imjared/5201405

http://casperjs.readthedocs.org/en/latest/modules/casper.html#evaluate

http://blog.csdn.net/u012577500/article/details/18185399

http://*.com/questions/14894311/casperjs-windows-installation-how-is-it-done-the-correct-way-please

http://blog.csdn.net/sagomilk/article/details/20800543