Python爬虫实例实现抓取JD 搜索页数据
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests
import csv
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
def download(url, headers, num_retries=3):
print("download", url)
try:
response = requests.get(url, headers=headers)
print(response.status_code)
if response.status_code == 200:
return response.content
return None
except RequestException as e:
print(e.response)
html = ""
if hasattr(e.response, 'status_code'):
code = e.response.status_code
print('error code', code)
if num_retries > 0 and 500 <= code < 600:
html = download(url, headers, num_retries - 1)
else:
code = None
return html
def find_Computer(url, headers):
r = download(url, headers=headers)
page = BeautifulSoup(r, "lxml")
all_items = page.find_all('li', attrs={'class':'gl-item'})
with open("Computer.csv", 'w', newline='') as f:
writer = csv.writer(f)
fields = ('ID', '名称', '价格')
writer.writerow(fields)
for all in all_items:
Computer_id = all["data-sku"]
print(f"电脑ID为:{Computer_id}")
Computer_name = all.find('div', attrs={'class':'p-name p-name-type-2'}).find('em').text
print(f"电脑的名称为:{Computer_name}")
Computer_price = all.find('div', attrs={'class':'p-price'}).find('i').text
print(f"电脑的价格为:{Computer_price}元")
row = []
row.append(Computer_id)
row.append(Computer_name)
row.append(str(Computer_price) + "元")
writer.writerow(row)
def main():
headers = {
'User-agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
"referer": "https://passport.jd.com"
}
URL = "https://search.jd.com/Search?keyword=macbook%20pro&enc=utf-8&suggest=5.def.0.V09&wq=mac&pvid=d3b040ed68154b06a9ff9532a10b4cac"
find_Computer(URL, headers=headers)
if __name__ == '__main__':
main()
本文地址:https://blog.csdn.net/Teamoping/article/details/108234985