做了个简单的post请求爬虫，爬取广东省科技厅关于创新的新闻

朋友的课程设计需要用到数据，叫我帮个忙，好久没做爬虫了，有些生了，当时分析网页发现是post请求，然后去找了请求的地址，发现from-data要传的数据没多少，发生变化的只有keywords跟page，那简单了，你要什么，我就给你什么，返回json数据，loads成字典，拿到详情页url，再requests发get请求，响应结果用lxml的etree.HTML转换给xpah提取数据，数据直接插入数据库，好了。

import requests
import json
import re
import time
from pymysql import connect

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36'}
url = "http://search.gd.gov.cn/api/search/all"
def run():
    for i in range(1, 220):
        d = {'gdbsDivision': '440000',
             'gdbsOrgNum': '006939801',
            'keywords': '创新',
            'page': i,
            'position': 'title',
            'range': 'site',
            'service_area': 1,
            'site_id': '172',
            'sort': 'smart'}

        res = requests.post(url=url, headers=headers, data=d)
        res = json.loads(res.content.decode())
        data = res['data']['news']['list']
        # print(data)
        for b in data:
            title = b['title']
            title = str(re.sub('(</em>|<em>|&rdquo;|&ldquo;)', '', title))

            content = b['content']
            content = str(re.sub('(&ldquo;|&rdquo;|\n|\u3000)', '', content))

            link = b['url']
            # requests.get(headers=headers, url=link)

            publisher_src = b['publisher_src']

            k = c.cursor()
            try:
                sql = 'insert into data3(title,content,publisher_src,url) values("%s","%s","%s","%s");'%(title, content, publisher_src, link)
                k.execute(sql)
            except:
                print('出错')
            print(sql)
            k.close()
            c.commit()
    time.sleep(2)


if __name__ == '__main__':
    c = connect(host='localhost', user='root', password='root', port=3306, db='kj', charset='utf8')
    run()

拿走

本文地址：https://blog.csdn.net/qq_44657868/article/details/107449432