基于python爬虫的github-exploitdb漏洞库监控与下载
基于python爬虫的github-exploitdb漏洞库监控与下载
offensive.py(爬取项目历史更新内容)
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import re
import time
import urllib.request
import conf as cf
base_url = 'https://github.com/offensive-security/exploitdb/releases'
download_link_pattern = 'href="(.*?)zip" rel="nofollow">'
first_pattern = r'</span><a rel="nofollow" href="(.*?)">next.*'
page_pattern = r'>previous</a><a rel="nofollow" href="(.*?)">next.*'
class mycrawler:
def __init__(self, base_url=base_url, start_page="first 1 page"):
self.base_url = base_url
self.start_page = start_page
# self.headers = apache_request_headers();
# 对首页的爬取
def first_page(self):
try:
req = urllib.request.request(self.base_url)
html = urllib.request.urlopen(req)
doc = html.read().decode('utf8', 'ignore')
next_page = re.search(first_pattern, doc, re.m | re.i)
print('now working on page = {}\n'.format(self.start_page))
time.sleep(5)
self.fetch_download_link(self.base_url)
self.start_page = next_page.group(1)
# re.search(r'after = (.*?) ">next.*', next_page.group(1), re.m | re.i).group(1)
self.base_url = next_page.group(1)
# self.fetch_download_link(next_url)
except urllib.error.httperror as err:
print(err.msg)
self.fetch_next_page()
# 翻页
def fetch_next_page(self):
while true:
try:
req = urllib.request.request(self.base_url)
html = urllib.request.urlopen(req)
doc = html.read().decode('utf8', 'ignore')
next_page = re.search(page_pattern, doc, re.m | re.i)
print('now working on page {}\n'.format(self.start_page))
time.sleep(5)
#翻页时等待5秒
self.fetch_download_link(self.base_url)
self.start_page = next_page.group(1)
# re.search(r'after = (.*?) ">next.*', next_page.group(1), re.m | re.i).group(1)
self.base_url = next_page.group(1)
# self.fetch_download_link(next_url)
except urllib.error.httperror as err:
print(err.msg)
break
# 文件下载：将下载链接存到文件中
def fetch_download_link(self, aurl):
f = open('result.txt', 'a')
req = urllib.request.request(aurl)
html = urllib.request.urlopen(req)
doc = html.read().decode('utf8')
alist = list(set(re.findall(download_link_pattern, doc)))
for item in alist:
url = "https://github.com/" + item + "zip"
print('storing {}'.format(url))
f.write(url + '\n')
time.sleep(7)
f.close()
def run(self):
self.fetch_download_link()
if __name__ == '__main__':
mc = mycrawler()
mc.first_page()
text.py(监控首页更新，并爬取)
#!/usr/bin/env python
# -*- coding:utf-8 -*
from selenium import webdriver
import re
import time
import urllib.request
import conf as cf
base_url = 'https://github.com/offensive-security/exploitdb/releases'
download_link_pattern = 'href="(.*?)zip" rel="nofollow">'
first_pattern = r'</span><a rel="nofollow" href="(.*?)">next.*'
# 监控项目首页更新
def jiankong_page():
print("star monitoring ")
req = urllib.request.request(base_url)
html = urllib.request.urlopen(req)
doc = html.read().decode('utf8', 'ignore')
next_page = re.search(first_pattern, doc, re.m | re.i)
flag_page = next_page.group(1)
flag_list = []
# 首次抓取首页项目url
alist = list(set(re.findall(download_link_pattern, doc)))
for item in alist:
url = "https://github.com/" + item + "zip"
flag_list.append(url)
# 定时扫描监控（5h/次）
while true:
try:
time.sleep(5 * 60* 60)
req = urllib.request.request(base_url)
html = urllib.request.urlopen(req)
doc = html.read().decode('utf8', 'ignore')
next_page = re.search(first_pattern, doc, re.m | re.i)
# 判断翻页链接是否变化，来确定是否更新
if next_page.group(1) != flag_page:
print("have update")
item = re.rearch(download_link_pattern, doc, re.m | re.i)
#抓取第一个匹配的 刚更新的项目url
new_url = "https://github.com/" + item.group(1) + "zip"
print("new url = " + new_url)
flag_list.append(new_url)
f = open('result.txt', 'a')
f.write(new_url + '\n')
f.close()
flag_page = next_page.group(1)
else:
print("no update")
except urllib.error.httperror as err:
print(err.msg)
break
if __name__ == '__main__':
jiankong_page()
介绍一下我自己吧，我是fisher，互联网安全作者一枚，日常是分享有趣的安全技术与故事，当然也会记录学习之路的收获。对安全领域感兴趣，可以关注我的个人微信公众号：austfish。不想走丢的话，请关注【fisher的安全日记】！（别忘了加星标哦）or 个人博客：