欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

python爬虫爬取网站小说

程序员文章站 2022-06-05 18:46:41
...
#加载模块
import requests
from bs4 import BeautifulSoup

# 定义所有章节和链接函数
def get_novel_chapters():
    url = "https://www.89wxw.com/read/1037/"
    r = requests.get(url)
    main_page = BeautifulSoup(r.text, "html.parser")
    list1 = []
    for dd in (main_page.find_all("dd")):
        link = dd.find("a")
        if not link:
            continue
        list1.append(("https://www.89wxw.com" + (link["href"]), (link.get_text())))
    return list1

# 定义获取所有章节内容
def get_chapters_content(url):
    r = requests.get(url)
    main_page = BeautifulSoup(r.text, "html.parser")
    data = main_page.find("div", id="content").get_text()
    return data

#循环调用两个自定义函数将鸟啼和内容写入文本文档
j = 0
f = open("都市超级医圣.txt", "w")
alist = get_novel_chapters()
for i in alist:
    j+=1
    url,title=i
    print("循环爬取第{}次,写入文本文档,标题是:{}".format(j,title))
    f.write(title+"\n"+get_chapters_content(url))
f.close()