猫眼电影爬取

headers设置

user_agent='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'
cookie='__mta=45932315.1608295245480.1609932302195.1609932384970.73; uuid_n_v=v1; _lxsdk_cuid=17675dde089c8-07a468a14a4d2b-59442e11-144000-17675dde089c8; __mta=45932315.1608295245480.1608345693705.1608347288332.39; uuid=23932090420011EB938EB13948D660ED4C5735DFBB854F1482C8F32A7D2A939F; _lxsdk=23932090420011EB938EB13948D660ED4C5735DFBB854F1482C8F32A7D2A939F; _csrf=0ad5150c37ad8b0d35bfa75267641e55060d7a0395b0a070219bb914e540fd37; Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1608385740,1608450456,1608536146,1609932146; Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1609932385; _lxsdk_s=176d76f040c-1b4-538-b6b%7C%7C8'
headers={"user-agent":user_agent,"cookie":cookie}

数据爬取

server=[]
for i in range(10):
    url="https://maoyan.com/board/4?offset="+str(i*10)
    cat=requests.get(url,headers=headers,proxies = {"http":"58.253.157.136"})
    cat_text=cat.text
    server.append(cat.text)

建立空表

maoyan=pd.DataFrame()

信息提取

for i in server:
    data=etree.HTML(i)
    pos=data.xpath('//dl[@class="board-wrapper"]')
    for j in pos:
        ranking=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/i/text()')
        picture=j.xpath('.//a/img/@data-src')
        movie_web=j.xpath('.//div/div[1]/div[1]/p[1]/a/@href')
        movie_name=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[1]/a/text()')
        performer=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[2]/text()')
        Release_date=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[1]/p[3]/text()')
        grades1=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[2]/p/i[1]/text()')
        grades2=j.xpath('//*[@id="app"]/div/div/div[1]/dl/dd/div/div/div[2]/p/i[2]/text()')
        score=[]
        for h in range(0,len(grades1)):
            score.append(grades1[h]+grades2[h])
        result=pd.DataFrame({"电影排名":ranking,
                             "图片地址":picture,
                             "电影详情页地址":movie_web,
                             "电影名称":movie_name,
                             "电影主演":performer,
                             "首映时间":Release_date,
                             "评分":score})
        maoyan=maoyan.append(result,ignore_index=True)

补全

maoyan['详情页'] ='https://maoyan.com'+maoyan['电影详情页地址']
del(maoyan['电影详情页地址'])

导出

maoyan.to_excel("猫眼数据top100爬取.xlsx",index=False)