爬取学校新闻标题

爬取学校新闻标题

# -*- coding: utf-8 -*-
import requests
from lxml import etree

if __name__ == '__main__':
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
                             ' (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'}
    #爬取页面源码数据
    url = 'http://www.lzy.edu.cn/index/xxxw/%d.htm'
    # 创建TXT文件存储数据
    fp=open('biaoti.txt','w',encoding='utf-8')
    # 分页操作，爬取1~122的数据
    for pageNum in range(1,122):
        new_url = format(url%pageNum)
        # 对单独一页发送请求
        page=requests.get(url=new_url,headers=headers)
    # with open('shuju.txt','w',encoding='utf-8') as fp:
    #     fp.write(page_text)
    # print(page_text)
    #防止乱码
        page.encoding = 'utf-8'
        page_text = page.text
    #print(page_text)
    #解析数据
        tree = etree.HTML(page_text)
    #找到所需数据的位置
        li_list = tree.xpath('//ul[@class="mainBlc"]/li')
        for li in li_list:
            #提取所需的数据
            title=li.xpath('./span/a/text()')[0]
            # 储存数据
            fp.write(title+'\n')
            print(title)