欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

Scrapy学习第八课

程序员文章站 2022-07-09 23:30:10
...

目标爬取文章,实现文本和图片数据存储

文本数据以json文件存储

文本数据存储在mongodb数据库中

图片保存在本地

爬取地址:伯乐在线文章

爬虫实例

1.items.py

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class JobboleItem(scrapy.Item):
    #标题
    title = scrapy.Field()

    #发布日期
    create_date = scrapy.Field()

    #链接
    url = scrapy.Field()

    #MD5加密的url
    url_object_id = scrapy.Field()

    #图片的url
    front_image_url = scrapy.Field()

    #图片存储路径
    front_image_path = scrapy.Field()

    #点赞数
    praise_nums = scrapy.Field()

    #收藏数
    fav_nums = scrapy.Field()

    #评论数
    comment_nums = scrapy.Field()

    #标签
    tag = scrapy.Field()

    #内容
    #content = scrapy.Field()

2.bole.py

# -*- coding: utf-8 -*-
import scrapy
from urllib.parse import urljoin
from jobBole.items import JobboleItem
import re
import hashlib
import datetime

def get_md5(md5str):
    #生成1个MD5对象
    m1 = hashlib.md5()
    #使用MD5对象你的update方法进行md5转换
    m1.update(md5str.encode("utf-8"))
    md5ConvertStr = m1.hexdigest()
    return md5ConvertStr


class BoleSpider(scrapy.Spider):
    name = 'bole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        '''
        1.获取文章列表也中具体文章url,并交给scrapy进行下载后并进行解析
        2.获取下一页的url并交给scrapy进行下载,下载完成后,交给parse
        :param response:
        :return:
        '''

        #解析列表页中所有文章的url, 并交给scrapy下载并解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            #image_url是图片的地址
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            #这里通过meta参数将图片的url传递进来,parse.urljoin的好处是如果有域名,则前面的response.url不生效
            #如果没有,就会把response.url和post_urlz做拼接
            yield scrapy.Request(url=urljoin(response.url, post_url), meta={
                "front_image_url": urljoin(response.url, image_url)
            },callback = self.parse_detail)  

        #提取下一页并交给scrapy下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")   
        curr_page = int(response.xpath('//span[@class="page-numbers current"]/text()').extract()[0])
        if next_url and curr_page < 3:
            yield scrapy.Request(url = next_url, callback = self.parse)

    def parse_detail(self, response):
        '''
        获取文章的详细内容
        :param response:
        :return:
        '''
        article_item = JobboleItem()
        front_image_url = response.meta.get("front_image_url", "")
        title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first()
        
        create_date = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract()[0].strip().split()[0]
        tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
        #去掉标签中的评论
        tag_list = [element for element in tag_list if  -1 == element.find("评论")]
        tag = ",".join(tag_list)
        praise_nums = response.xpath('//span[contains(@class, "vote-post-up")]/h10/text()') .extract()[0]
        print('praise_nums  ',  praise_nums)
        if len(praise_nums) == 0:
            praise_nums = 0
        else:
            praise_nums = int(praise_nums[0]) 
       
        fav_nums = response.xpath('//span[contains(@class, "bookmark-btn")]/text()').extract()[0]
        match_re = re.match(".*(\d+).*", fav_nums)
        if match_re:
            fav_nums = int(match_re.group(1))
        else:
            fav_nums = 0
        #print('@@@@   ', response.xpath('//a[@href="#article-comment"]/span/text()').extract())
        comment_nums = response.xpath('//a[@href="#article-comment"]/span/text()').extract()[0]
    
        match_com = re.match(".*(\d+).*", comment_nums)
        if match_com:
            comment_nums= int(match_com.group(1))
        else:
            comment_nums = 0

        content = response.xpath('//div[@class="entry"]').extract()[0]

        article_item['url_object_id'] = get_md5(response.url) #对地址进行md5变成了定长
        article_item['title'] = title
        article_item['url'] = response.url
        try:
            create_date = datetime.datetime.strptime(create_date, '%Y/%m/%d').date()
        except Exception as e:
            create_date = datetime.now().date()

        article_item['create_date'] = str(create_date)
        article_item['front_image_url'] = [front_image_url]
        article_item['praise_nums'] = int(praise_nums)
        article_item['fav_nums'] = fav_nums
        article_item['comment_nums'] = comment_nums
        article_item['tag'] = tag
        #article_item['content'] = content
        yield article_item

3.settings.py

ITEM_PIPELINES = {
   'jobBole.pipelines.JobbolePipeline': 300,
   'jobBole.pipelines.ArticleImagePipeline' : 301,
   'jobBole.pipelines.MongoDBTwistedPipline': 302
}
IMAGES_STORE = 'D:\SunWork\python\jobBole'

MONGODB_HOST = '127.0.0.1'
MONGODB_PORT = 27017
MONGODB_DBNAME = 'bole'
MONGODB_SHEETNAME = 'bolePaper'

4.pipelines.py

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import scrapy
from scrapy.pipelines.images import ImagesPipeline
import codecs
import json
import pymongo
from scrapy.conf import settings

class JobbolePipeline(object):
    '''
    返回json数据到文件中
    '''
    def __init__(self):
        self.file = codecs.open("article.json", 'w',encoding='utf-8')

    def process_item(self, item, spider):
        print('@@@@@@@@@@ ', item)
        lines = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.close()

class ArticleImagePipeline(ImagesPipeline):
    '''
    对图片的处理
    '''
    def get_media_requests(self, item, info):
        for image_url in item['front_image_url']:
            yield scrapy.Request(image_url)
        

    def item_completed(self, results, item, info):
        for ok, value in results:
            if ok:
                image_file_path = value['path']
                item['front_image_path'] = image_file_path
            else:
                item['front_image_path'] = ""
           
        return item
    
class MongoDBTwistedPipline(object):
    def __init__(self):
        #主机
        host = settings["MONGODB_HOST"]
        #端口
        port = settings["MONGODB_PORT"]
        #数据库名
        dbname = settings["MONGODB_DBNAME"]
        #数据表名
        sheetname = settings["MONGODB_SHEETNAME"]
        #创建MONGODB数据库
        client = pymongo.MongoClient(host=host, port=port)
        #指定数据库
        mydb = client[dbname]
        #指定数据表
        self.post = mydb[sheetname]
    
    def process_item(self, item, spider):
        data = dict(item)
        self.post.insert(data)
        return item

注:代码来源https://www.cnblogs.com/zhaof/p/7173094.html。在此基础上进行部分修改。