python爬虫爬取网站小说
程序员文章站
2022-06-05 18:46:41
...
#加载模块
import requests
from bs4 import BeautifulSoup
# 定义所有章节和链接函数
def get_novel_chapters():
url = "https://www.89wxw.com/read/1037/"
r = requests.get(url)
main_page = BeautifulSoup(r.text, "html.parser")
list1 = []
for dd in (main_page.find_all("dd")):
link = dd.find("a")
if not link:
continue
list1.append(("https://www.89wxw.com" + (link["href"]), (link.get_text())))
return list1
# 定义获取所有章节内容
def get_chapters_content(url):
r = requests.get(url)
main_page = BeautifulSoup(r.text, "html.parser")
data = main_page.find("div", id="content").get_text()
return data
#循环调用两个自定义函数将鸟啼和内容写入文本文档
j = 0
f = open("都市超级医圣.txt", "w")
alist = get_novel_chapters()
for i in alist:
j+=1
url,title=i
print("循环爬取第{}次,写入文本文档,标题是:{}".format(j,title))
f.write(title+"\n"+get_chapters_content(url))
f.close()
上一篇: C#类型成员:方法
下一篇: 等白菜500块一斤的时候