【python爬虫专项（18）】基于爬虫的MongoDB的应用（将爬取的数据存入到数据库）

程序员文章站 2022-05-03 13:13:50

...

之前在去哪儿网采集的景点数据，添加函数将数据存入mongo

函数：get_data(ui,d_h,d_c,table) → 【数据采集及mongo入库】
       ui：数据信息网页
       d_h：user-agent信息
       d_c：cookies信息
       table：mongo集合对象
       结果：每采集一条数据，存入mongo数据库

全部代码：
主要是针对于get_data()函数里面的部分代码进行修改，按一条条数据写入数据库的方式运行代码，当然在处理之前需要先导入pymongo和预先设置好储存数据的地方，最后为了可视化输出和保证程序正常运行。可以加上数据采集和储存的进度情况以及异常判断的操作

import requests 
from bs4 import BeautifulSoup
import pandas as pd
import pymongo

def get_url(n):
	'''
	【分页网址url采集】函数
	n：页数参数
	结果：得到一个分页网页的list
	'''
	lst = []
	for i in range(n):
		ui = "https://travel.qunar.com/p-cs299878-shanghai-jingdian-1-{}".format(i+1)
		lst.append(ui)
	return lst

def get_data(ui,d_h,d_c,table):  
	'''
	【数据采集及mongo入库】
	ui：数据信息网页
	d_h：user-agent信息
	d_c：cookies信息
	结果：得到数据的list，每条数据用dict存储
	'''
	ri = requests.get(ui, headers= dic_heders, cookies = dic_cookies)
	soup_i = BeautifulSoup(ri.text,'lxml')
	ul = soup_i.find("ul",class_="list_item clrfix")
	lis = ul.find_all('li')
	
	#这里删除了添加到列表的操作，直接将生成的字典添加到mongo，同时增加了计数
	n = 0
	for li in lis:
		dic = {}
		dic['景点名称'] = li.find('span',class_="cn_tit").text
		dic['攻略数量'] = li.find('div',class_="strategy_sum").text
		dic['评分'] = li.find('span',class_="total_star").span['style']
		dic['简介'] = li.find('div',class_="desbox").text
		dic['排名'] = li.find('span',class_="ranking_sum").text
		dic['经度'] = li['data-lng']
		dic['纬度'] = li['data-lat']
		dic['点评数量'] = li.find('div',class_="comment_sum").text
		dic['多少驴友来过'] = li.find('span',class_="comment_sum").span.text
		table.insert_one(dic)
		n += 1
	return n


if __name__ == "__main__":

	dic_heders = {
	   'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
	}

	dic_cookies = {}
	cookies = 'QN1=dXrgj14+tmYQhFxKE9ekAg==; QN205=organic; QN277=organic; QN269=506F28C14A7611EAA0BEFA163E244083; _i=RBTKSRDqFhTQT5KRlx-P1H78agxx; fid=7cc3c3d9-3f6c-45e1-8cef-3384cd5da577; Hm_lvt_c56a2b5278263aa647778d304009eafc=1581168271,1581220912; viewpoi=7564992|709275; viewdist=299878-7; uld=1-299878-8-1581221233|1-1062172-1-1581168529; QN267=1679639433d5aedfc8; Hm_lpvt_c56a2b5278263aa647778d304009eafc=1581221236; QN25=cb06bfbd-d687-4072-98c5-73266b637a6a-9f992f90; QN42=nvxp8441; _q=U.qunar_lbs_428305502; _t=26463150; csrfToken=oXYBnhSoGAGxggRkzmAjbxxGrpgsjUqQ; _s=s_ZBWFJO3EEGZISWS35EBIS5NQYA; _v=YTRjW_H5L47nGNVabvTLt1mlh7j8R7t4UNDVRrJUz0wScfLMWgSvkwQbzMLHlFbsvTU-2kJrBK74NUyOi3MX_3obY94Hhhugt8bv8ILxwsWDv4s_ANNiM8qRdg6HlBrrCEnGYr8lxS9uv78zDCNKz9pFbN8JPYy-AKJP6xILIsT7; _vi=4ONQzvfOOhwJECN5R-4rfWZDzlQ5-qv2xi_jsp1INPEpy9iKHa5gV0gHc35fDfTDe3TjcKteU7ZWk1vd6MsIqTfXYyUh3gTwZJ_9z3PEpkXZReeeIjaVE4HwLTkOATLIzIxg92s-QCWKE1RdNlaZsxPnfN7NHPGAZz5rsmxvpNDY; QN44=qunar_lbs_428305502; QN48=tc_a7fe4861b2d918df_17028369fc8_67ab; QN271=1749d44a-1a11-4886-be27-c3e3bfdadb0c'
	cookies_lst = cookies.split("; ")
	for i in cookies_lst:
		dic_cookies[i.split("=")[0]] = i.split("=")[1]    

	#导入数据库，并分配数据存储位置
	myclient = pymongo.MongoClient("mongodb://localhost:27017/")
	db = myclient['去哪儿网数据']
	datatable = db['data']
	
	urllst = get_url(10)

	errorlst =[]
	count = 0
	for u in urllst:
		try:
			count += get_data(u,dic_heders,dic_cookies,datatable)
			print('数据采集并存入成功，总共采集{}条数据'.format(count))
		except:
			errorlst.append(u)
			print('数据采集失败，数据网址为：',u)