Scrapy之携带cookie请求登录

spiders中爬虫.py

# -*- coding: utf-8 -*-
import scrapy
import re
 
 
class RenrenSpider(scrapy.Spider):
    name = '爬虫名'
    allowed_domains = ['renren.com']
    start_urls = ['http://www.renren.com/327550029/profile']  # 需要登陆后才能正常访问的url
 
    # start_urls都是默认交给start_requests函数处理的，可以重写该方法添加请求头、cookie等。
    def start_requests(self):
        # 该cookie是登陆后获取的cookie字符串。
        cookies = "anonymid=jcokuqturos8ql; depovince=GW; jebecookies=f90c9e96-78d7-4f74-b1c8-b6448492995b|||||; _r01_=1; JSESSIONID=abcx4tkKLbB1-hVwvcyew; ick_login=ff436c18-ec61-4d65-8c56-a7962af397f4; _de=BF09EE3A28DED52E6B65F6A4705D973F1383380866D39FF5; p=90dea4bfc79ef80402417810c0de60989; first_login_flag=1; [email protected]; ln_hurl=http://hdn.xnimg.cn/photos/hdn421/20171230/1635/main_JQzq_ae7b0000a8791986.jpg; t=24ee96e2e2301bf2c350d7102956540a9; societyguester=24ee96e2e2301bf2c350d7102956540a9; id=327550029; xnsid=e7f66e0b; loginfrom=syshome; ch_id=10016"
        # 转换成字典
        cookies = {i.split("=")[0]:i.split("=")[1] for i in cookies.split("; ")}
        # headers = {"Cookie":cookies}  # cookie放到headers中无效
        yield scrapy.Request(
            self.start_urls[0],
            callback=self.parse,
            cookies=cookies   # 携带cookie进行请求
            # headers = headers  # cookie放到headers中无效
        )
 
    def parse(self, response):
        ......
        # 后续的请求会自动携带之前的cookie (setting.py配置文件中不要禁用cookie)
        yield scrapy.Request(
            "http://www.renren.com/327550029/profile?v=info_timeline",
            callback=self.parse_detial
        )
 
    def parse_detial(self,response):
       .....

settings.py（配置文件，不要禁用cookie）

# 禁用cookie (默认开启cookie)
# COOKIES_ENABLED = False
COOKIES_DEBUG = True  # 开启cookie的调试信息