欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

requsets的过程

程序员文章站 2022-05-03 23:45:37
...
因为requests基于urllib,所以看一下urllib的请求过程。
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2


# python2
def load_baidu_data():
    url = 'http://www.baidu.com'
    request = urllib2.Request(url)  # 生成request实例
    response = urllib2.urlopen(request)
    data =response.read()
    return data

if __name__ == '__main__':
    print(load_baidu_data())
window,python2中的urllib get请求。
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2


# 自定义urlopen
def user_defined():
    # url
    url = "http://www.baidu.com"
    # headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # request
    request = urllib2.Request(url, headers=headers)

    # 1.创建处理器
    handler = urllib2.HTTPHandler()

    # 2.根据处理器 自定义openner
    opener = urllib2.build_opener(handler)

    # 3.open()发送请求
    response = opener.open(request)

    print response.read()

if __name__ == '__main__':
    user_defined()
实际的过程是创建处理器HTTPHandler,根据处理器自定义opener,根据opener发送请求open。

那有用户代理呢
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2

# 创建 具有 代理IP功能的处理器


def proxy_opener():

    # 1.url
    url = "http://www.baidu.com"
    # 2.headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # 3.request
    request = urllib2.Request(url, headers=headers)

    # 代理ip
    proxy = {"http": ":"}

    # 1.创建具有 proxy处理器
    proxy_handler = urllib2.ProxyHandler(proxy)

    # 2.根据处理器创建opener
    opener = urllib2.build_opener(proxy_handler)

    # 4.open()发送请求
    response = opener.open(request)
    # 5.返回数据
    print response.read()

if __name__ == '__main__':
    proxy_opener()

实际的过程是创建具有代理功能的处理器ProxyHandler,根据处理器自定义opener,根据opener发送请求open。

web认证
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2


def web_auth_open():
    url = ''
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    request = urllib2.Request(url, headers=headers)
    # 密码管理器
    pwd_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
    # 添加用户名密码
    pwd_manager.add_password(None, uri=url, user='', passwd='')
    # 认证处理器
    web_handler = urllib2.HTTPBasicAuthHandler(pwd_manager)
    # 自定义opener
    web_opener = urllib2.build_opener(web_handler)
    # 自定义open
    response = web_opener.open(request)

    print response.read()

if __name__ == '__main__':
    web_auth_open()
实际的过程是,创建密码管理器,创建具有认证功能的处理器HTTPBasicAuthHandler,根据处理器自定义opener,根据opener发送请求open。
cookie
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve
import urllib2

import urllib

import cookielib


# 测试人人网
def cookie_open():
    login_url = ''
    data_url = ""
    # headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"
               }
    formdata = {
        "email": "",
        "password": ""
    }
    formdata = urllib.urlencode(formdata)
    login_request = urllib2.Request(login_url, headers=headers, data=formdata) #post
    data_request = urllib2.Request(data_url,headers=headers) # get
    # cookie罐子
    cookie_jar = cookielib.CookieJar()
    # cookie处理器
    cookie_handler = urllib2.HTTPCookieProcessor(cookiejar=cookie_jar)
    # cookie opener
    cookie_opener = urllib2.build_opener(cookie_handler)
    # cookie open 先登录
    cookie_opener.open(login_request)
    # 后去数据页
    data_response = cookie_opener.open(data_request)
    data = data_response.read()
    print data

if __name__ == '__main__':
    data = cookie_open()
实际的过程是,创建cookie_jar,创建具有cookie功能的处理器HTTPCookieProcessor,根据处理器自定义opener,根据opener发送请求open。

忽略ssl认证
#!/usr/bin/env python
# -*- coding:utf-8 -*- 
# Author: json_steve

import urllib2

import ssl

#如果 网站没有CA证书 访问受限制  可以 告诉系统忽略

def ssl_load_data():

    # 1.url
    url = "https://www.12306.cn/mormhweb/"
    # 2.headers
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}

    # 3.request
    request = urllib2.Request(url,headers=headers)

    # 4.发送请求
    #主动告诉系统 忽略证书ssl
    context = ssl._create_unverified_context()
    resposne = urllib2.urlopen(request,context=context)

    with open("","w") as f:
        f.write(resposne.read())


if __name__ == '__main__':
    ssl_load_data()
相关标签: urllib