免费的代理IP池

# encoding=utf-8
#构建免费的代理IP池这里我们使用西拉代理,代理云,快代理
import telnetlib
import urllib
import logging
import requests
import time
# ------------------------------------------
#   版本：1.0
#   日期：2019-10-09
#   作者：DengTing
 
#   <e-mail: [email protected]>
# ------------------------------------------

logger = logging.getLogger(__name__)
IPPOOLNUM=30 #一次性从网页获取的IP数量

def GetIPPOOLS():
    #西拉代理
    # url = 'http://www.xiladaili.com/api/?uuid=d426e85f09294a0bacde334d27d04297&num=50&protocol=0&sortby=2&repeat=2&format=3&position=1'
    #代理云
    url = 'http://huzi1020.v4.dailiyun.com/query.txt?key=NPCAD43ADF&word=&count=300&rand=false&detail=false'
    #快代理
    url = 'https://dps.kdlapi.com/api/getdps/?orderid=975192672891280&num=50&pt=1&sep=3'
    user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
    headers = {'User-Agent':user_agent}
    resp = requests.get(url, headers=headers)

    ret = [ x.strip() for x in resp.text.split('\r\n') if x ]
    # print(ret)
    # time.sleep(3)
    # dic = json.loads(resp.text)
    # ret = [ x['ip'] for x in dic['data'] ]
    return ret
    
def initIPPOOLS(rconn):

    """把有效的IP存入	REDIS数据库"""

    ipNum=len(rconn.keys('IP*'))
    if ipNum<IPPOOLNUM:
        IPPOOLS=GetIPPOOLS()
        for ipall in IPPOOLS:
            try:
                ip=ipall.split(':')[0]
                port=ipall.split(':')[1]
                telnetlib.Telnet(ip,port=port,timeout=2)
            except:
                logger.warning("The ip is not available !( IP:%s )" % ipall)
            else:
                logger.warning("Get ip Success!( IP:%s )" % ipall)
                rconn.set("IP:%s"%(ipall),ipall)  
                rconn.expire("IP:%s"%(ipall),540) 
    else:
        logger.warning("The number of  the IP is %s!" % str(ipNum))

def removeIPPOOLS(rconn,ip):
    logger.error("IP:%s not available ! System is deleting" % ip)
    try:
        rconn.delete('IP:' + ip)
    except:
        pass
    ipNum=len(rconn.keys('IP*'))
    logger.warning("The number of  the IP is %s!" % str(ipNum))