爬虫实战(1)——爬取中国天气网并进行简单可视化
程序员文章站
2022-07-14 16:50:18
...
import requests
from bs4 import BeautifulSoup
from pyecharts.charts import Bar
from pyecharts import options as opts
ALL_data=[]
def parse_url(url):
headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"}
response=requests.get(url,headers=headers)
text=response.content.decode("utf-8")
soup=BeautifulSoup(text,'html5lib')#html5lib容错性很强
divs=soup.find("div",class_="conMidtab")
tables=divs.find_all("table")
#print(tables)
for table in tables:
trs=table.find_all("tr")[2:]
for index,tr in enumerate(trs):
tds=tr.find_all("td")
if index == 0:
city_name=tds[1]
city=list(city_name.stripped_strings)[0]
else:
city_name=tds[0]
city=list(city_name.stripped_strings)[0]
temp=tds[-2]
min_temp=list(temp.stripped_strings)[0]
ALL_data.append({'city':city,'min_temp':int(min_temp)})
def main():
urls=["http://www.weather.com.cn/textFC/gat.shtml#",
"http://www.weather.com.cn/textFC/hb.shtml#",
"http://www.weather.com.cn/textFC/db.shtml#"]
for url in urls:
parse_url(url)
ALL_data.sort(key=lambda data:data['min_temp'])
data=ALL_data[0:10]
cities=list(map(lambda x:x['city'],data))
temps=list(map(lambda x:x['min_temp'],data))
bar = (
Bar()
.add_xaxis(cities)
.add_yaxis('',temps)
.set_global_opts(title_opts=opts.TitleOpts(title="中国天气最低气温排行"))
)
bar.render("weather.html")
main()
上一篇: opencv 泊松融合