Scrapy-从数据库取出IP并判断是否可用
程序员文章站
2024-04-02 22:35:58
```python import pymysql import requests conn = pymysql.connect(host="localhost",user="root",passwd="root",db="CDL",charset="utf8") cursor = conn.curs ......
import pymysql import requests conn = pymysql.connect(host="localhost",user="root",passwd="root",db="cdl",charset="utf8") cursor = conn.cursor() class iputils(object): # 删除无效ip def delete_ip(self,ip): sql ="delete from xiciproxy where ip='{0}'".format(ip) cursor.execute(sql) conn.commit() return true # 判断ip是否可用 def judge_ip(self,type,ip,port): http_url ="http://www.baidu.com" proxy_url ="{0}://{1}:{2}".format(type,ip,port) print("url:"+ proxy_url) try: proxy_dict = { type: proxy_url, # type 为https 或者http 数据库存储的 } response = requests.get(http_url,proxies=proxy_dict) exceptexceptionase: self.delete_ip(ip) print("invalid ip and port") return false code = response.status_code if code >=200 and code <300: print("effective ip") return true else: print("invalid ip and port") self.delete_ip(ip) return false # 随机获取ip def get_random_ip(self): random_sql =""" select type, ip, port from xiciproxy order by rand() limit 1 """ result = cursor.execute(random_sql) for ip_info in cursor.fetchall(): type = ip_info[0] ip = ip_info[1] port = ip_info[2] judge_re =self.judge_ip(type,ip,port) ifjudge_re: return"{0}://{1}:{2}".format(type,ip,port) else: return self.get_random_ip() if__name__ =="__main__": ip = iputils() print(ip.get_random_ip())