MasonLiu/github-cve-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复N多BUG，去除日志模块，目前基本稳定

Browse Source 
修复N多BUG，去除日志模块，目前基本稳定
This commit is contained in:
wuyoukm 2021-09-09 16:42:53 +08:00 committed by GitHub
parent 9e2cfcdada
commit e9bc94dc35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
github_cve_monitor.py
View File

@ -17,10 +17,6 @@ import hashlib
import yaml import yaml
from lxml import etree from lxml import etree
import sqlite3 import sqlite3
import logging
logging.basicConfig(level=logging.DEBUG, filename='run_info.log')
#读取配置文件 #读取配置文件
def load_config(): def load_config():
@ -43,7 +39,6 @@ def load_config():
return app_name,github_token,tgbot_token,tgbot_group_id return app_name,github_token,tgbot_token,tgbot_group_id
elif int(config['all_config']['tgbot'][0]['enable']) == 0 and int(config['all_config']['server'][0]['enable']) == 0 and int(config['all_config']['dingding'][0]['enable']) == 0: elif int(config['all_config']['tgbot'][0]['enable']) == 0 and int(config['all_config']['server'][0]['enable']) == 0 and int(config['all_config']['dingding'][0]['enable']) == 0:
print("[-] 配置文件有误，三个社交软件的enable不能为0") print("[-] 配置文件有误，三个社交软件的enable不能为0")
logging.error("[-] 配置文件有误，三个社交软件的enable不能为0")
github_headers = { github_headers = {
'Authorization': "token {}".format(load_config()[1]) # 替换自己的github token https://github.com/settings/tokens/new 'Authorization': "token {}".format(load_config()[1]) # 替换自己的github token https://github.com/settings/tokens/new
@ -68,16 +63,13 @@ def create_database():
pushed_at varchar(255), pushed_at varchar(255),
cve_url varchar(255));''') cve_url varchar(255));''')
print("成功创建CVE监控表") print("成功创建CVE监控表")
logging.info('成功创建CVE监控表')
cur.execute('''CREATE TABLE IF NOT EXISTS redteam_tools_monitor cur.execute('''CREATE TABLE IF NOT EXISTS redteam_tools_monitor
(tools_name varchar(255), (tools_name varchar(255),
pushed_at varchar(255), pushed_at varchar(255),
tag_name varchar(255));''') tag_name varchar(255));''')
print("成功创建红队工具监控表") print("成功创建红队工具监控表")
logging.info('成功创建红队工具监控表')
except Exception as e: except Exception as e:
print("创建cve监控表失败！报错：{}".format(e)) print("创建cve监控表失败！报错：{}".format(e))
logging.error("创建cve监控表失败！报错：{}".format(e))
conn.commit() # 数据库存储在硬盘上需要commit 存储在内存中的数据库不需要 conn.commit() # 数据库存储在硬盘上需要commit 存储在内存中的数据库不需要
conn.close() conn.close()
if load_config()[0] == "dingding": if load_config()[0] == "dingding":
@ -103,19 +95,16 @@ def getNews():
try: try:
cve_name_tmp = json_str['items'][i]['name'].upper() cve_name_tmp = json_str['items'][i]['name'].upper()
cve_name = re.findall('(CVE\-\d+\-\d+)', cve_name_tmp)[0].upper() cve_name = re.findall('(CVE\-\d+\-\d+)', cve_name_tmp)[0].upper()
pushed_at_tmp = json_str['items'][i]['pushed_at']
pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
if pushed_at == str(today_date):
today_cve_info_tmp.append({"cve_name": cve_name, "cve_url": cve_url, "pushed_at": pushed_at})
else:
print("[-] 该{}的更新时间为{}，不属于今天的CVE".format(cve_name, pushed_at))
except Exception as e: except Exception as e:
pass pass
pushed_at_tmp = json_str['items'][i]['pushed_at']
pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
# today_cve_info_tmp.append({"cve_name": cve_name, "cve_url": cve_url, "pushed_at": pushed_at})
if pushed_at == str(today_date):
today_cve_info_tmp.append({"cve_name":cve_name,"cve_url":cve_url,"pushed_at":pushed_at})
else:
print("[-] 该{}的更新时间为{}，不属于今天的CVE".format(cve_name,pushed_at))
logging.info("[-] 该{}的更新时间为{}，不属于今天的CVE".format(cve_name,pushed_at))
else: else:
logging.info("该{}，属于黑名单用户{}提交的CVE".format(cve_name,cve_url.split("/")[-2])) pass
today_cve_info = OrderedDict() today_cve_info = OrderedDict()
for item in today_cve_info_tmp: for item in today_cve_info_tmp:
today_cve_info.setdefault(item['cve_name'], {**item, }) today_cve_info.setdefault(item['cve_name'], {**item, })
@ -127,20 +116,17 @@ def getNews():
except Exception as e: except Exception as e:
print(e, "github链接不通") print(e, "github链接不通")
logging.error(e, "github链接不通")
return '', '', '' return '', '', ''
#获取到的CVE信息插入到数据库 #获取到的CVE信息插入到数据库
def cve_insert_into_sqlite3(data): def cve_insert_into_sqlite3(data):
conn = sqlite3.connect('data.db') conn = sqlite3.connect('data.db')
print("cve_insert_into_sqlite3 函数 打开数据库成功！") print("cve_insert_into_sqlite3 函数 打开数据库成功！")
logging.info("cve_insert_into_sqlite3 函数 打开数据库成功！")
cur = conn.cursor() cur = conn.cursor()
for i in range(len(data)): for i in range(len(data)):
try: try:
cve_name = re.findall('(CVE\-\d+\-\d+)', data[i]['cve_name'])[0].upper() cve_name = re.findall('(CVE\-\d+\-\d+)', data[i]['cve_name'])[0].upper()
cur.execute("INSERT INTO cve_monitor (cve_name,pushed_at,cve_url) VALUES ('{}', '{}', '{}')".format(cve_name, data[i]['pushed_at'], data[i]['cve_url'])) cur.execute("INSERT INTO cve_monitor (cve_name,pushed_at,cve_url) VALUES ('{}', '{}', '{}')".format(cve_name, data[i]['pushed_at'], data[i]['cve_url']))
print("cve_insert_into_sqlite3 函数: {}插入数据成功！".format(cve_name)) print("cve_insert_into_sqlite3 函数: {}插入数据成功！".format(cve_name))
logging.info("cve_insert_into_sqlite3 函数: {}插入数据成功！".format(cve_name))
except Exception as e: except Exception as e:
pass pass
conn.commit() conn.commit()
@ -167,35 +153,28 @@ def get_today_cve_info(today_cve_info_data):
try: try:
today_cve_name = re.findall('(CVE\-\d+\-\d+)', today_cve_info_data[i]['cve_name'])[0].upper() today_cve_name = re.findall('(CVE\-\d+\-\d+)', today_cve_info_data[i]['cve_name'])[0].upper()
if exist_cve(today_cve_name) == 1: if exist_cve(today_cve_name) == 1:
logging.info("get_today_cve_info 函数 today_cve_name的值为 - > {}".format(today_cve_name))
Verify = query_cve_info_database(today_cve_name.upper()) Verify = query_cve_info_database(today_cve_name.upper())
if Verify == 0:
print("[+] 数据库里不存在{}".format(today_cve_name.upper()))
today_all_cve_info.append(today_cve_info_data[i])
else:
print("[-] 数据库里存在{}".format(today_cve_name.upper()))
except Exception as e: except Exception as e:
Verify = 1 pass
if Verify == 0:
print("[+] 数据库里不存在{}".format(today_cve_name.upper()))
logging.info("[+] 数据库里不存在{}".format(today_cve_name.upper()))
today_all_cve_info.append(today_cve_info_data[i])
else:
print("[-] 数据库里存在{}".format(today_cve_name.upper()))
logging.info("[-] 数据库里存在{}".format(today_cve_name.upper()))
return today_all_cve_info return today_all_cve_info
#获取红队工具信息插入到数据库 #获取红队工具信息插入到数据库
def tools_insert_into_sqlite3(data): def tools_insert_into_sqlite3(data):
conn = sqlite3.connect('data.db') conn = sqlite3.connect('data.db')
print("tools_insert_into_sqlite3 函数 打开数据库成功！") print("tools_insert_into_sqlite3 函数 打开数据库成功！")
logging.info("tools_insert_into_sqlite3 函数 打开数据库成功！")
cur = conn.cursor() cur = conn.cursor()
for i in range(len(data)): for i in range(len(data)):
Verify = query_tools_info_database(data[i]['tools_name']) Verify = query_tools_info_database(data[i]['tools_name'])
if Verify == 0: if Verify == 0:
print("[+] 红队工具表数据库里不存在{}".format(data[i]['tools_name'])) print("[+] 红队工具表数据库里不存在{}".format(data[i]['tools_name']))
logging.info("[+] 红队工具表数据库里不存在{}".format(data[i]['tools_name']))
cur.execute("INSERT INTO redteam_tools_monitor (tools_name,pushed_at,tag_name) VALUES ('{}', '{}','{}')".format(data[i]['tools_name'], data[i]['pushed_at'], data[i]['tag_name'])) cur.execute("INSERT INTO redteam_tools_monitor (tools_name,pushed_at,tag_name) VALUES ('{}', '{}','{}')".format(data[i]['tools_name'], data[i]['pushed_at'], data[i]['tag_name']))
print("tools_insert_into_sqlite3 函数: {}插入数据成功！".format(format(data[i]['tools_name']))) print("tools_insert_into_sqlite3 函数: {}插入数据成功！".format(format(data[i]['tools_name'])))
logging.info("tools_insert_into_sqlite3 函数: {}插入数据成功！".format(format(data[i]['tools_name'])))
else: else:
print("[-] 红队工具表数据库里存在{}".format(data[i]['tools_name'])) print("[-] 红队工具表数据库里存在{}".format(data[i]['tools_name']))
logging.info("[-] 红队工具表数据库里存在{}".format(data[i]['tools_name']))
conn.commit() conn.commit()
conn.close() conn.close()
#读取本地红队工具链接文件转换成list #读取本地红队工具链接文件转换成list
@ -220,7 +199,7 @@ def get_pushed_at_time(tools_list):
tag_name = "no releases" tag_name = "no releases"
tools_info_list.append({"tools_name":tools_name,"pushed_at":pushed_at,"api_url":api_url,"tag_name":tag_name}) tools_info_list.append({"tools_name":tools_name,"pushed_at":pushed_at,"api_url":api_url,"tag_name":tag_name})
except Exception as e: except Exception as e:
logging.error("get_pushed_at_time BUG -> {}".format(e)) pass
return tools_info_list return tools_info_list
#根据红队名名称查询数据库红队工具的更新时间以及版本名称并返回 #根据红队名名称查询数据库红队工具的更新时间以及版本名称并返回
@ -234,28 +213,21 @@ def tools_query_sqlite3(tools_name):
result_list.append({"pushed_at":result[0],"tag_name":result[1]}) result_list.append({"pushed_at":result[0],"tag_name":result[1]})
conn.close() conn.close()
print("[###########] tools_query_sqlite3 函数内 result_list 的值 为 - > {}".format(result_list)) print("[###########] tools_query_sqlite3 函数内 result_list 的值 为 - > {}".format(result_list))
logging.info("[###########] tools_query_sqlite3 函数内 result_list 的值 为 - > {}".format(result_list))
return result_list return result_list
#获取更新了的红队工具在数据库里面的时间和版本 #获取更新了的红队工具在数据库里面的时间和版本
def get_tools_update_list(data): def get_tools_update_list(data):
tools_update_list = [] tools_update_list = []
for dist in data: for dist in data:
print("dist 变量 ->{}".format(dist)) print("dist 变量 ->{}".format(dist))
logging.info("dist 变量 ->{}".format(dist))
query_result = tools_query_sqlite3(dist['tools_name']) query_result = tools_query_sqlite3(dist['tools_name'])
today_tools_pushed_at = query_result[0]['pushed_at'] today_tools_pushed_at = query_result[0]['pushed_at']
logging.info("[###########] get_tools_update_list 函数内 today_tools_pushed_at的值 - >{}".format(today_tools_pushed_at))
# print("[!!] 今日获取时间: ", dist['pushed_at'], "获取数据库时间: ", today_tools_pushed_at, dist['tools_name']) # print("[!!] 今日获取时间: ", dist['pushed_at'], "获取数据库时间: ", today_tools_pushed_at, dist['tools_name'])
if dist['pushed_at'] != today_tools_pushed_at: if dist['pushed_at'] != today_tools_pushed_at:
print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name'],"update!!!!") print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name'],"update!!!!")
logging.info("[!] tools_name: {} 今日API时间: {} 获取数据库时间: {} {}update!!!!".format(dist['tools_name'],dist['pushed_at'],today_tools_pushed_at,dist['tools_name']))
#返回数据库里面的时间和版本 #返回数据库里面的时间和版本
tools_update_list.append({"api_url":dist['api_url'],"pushed_at":today_tools_pushed_at,"tag_name":query_result[0]['tag_name']}) tools_update_list.append({"api_url":dist['api_url'],"pushed_at":today_tools_pushed_at,"tag_name":query_result[0]['tag_name']})
else: else:
print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name']," no update") print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name']," no update")
logging.info("[-] tools_name: {} 今日API时间: {} 获取数据库时间: {} {} no update".format(dist['tools_name'],dist['pushed_at'],today_tools_pushed_at,dist['tools_name']))
logging.info("get_tools_update_list 函数 tools_update_list数组的值为{}".format(tools_update_list))
# todo BUG在数组
return tools_update_list return tools_update_list
#获取更新信息并发送到对应社交软件 #获取更新信息并发送到对应社交软件
def send_body(url,query_pushed_at,query_tag_name): def send_body(url,query_pushed_at,query_tag_name):
@ -265,9 +237,8 @@ def send_body(url,query_pushed_at,query_tag_name):
new_pushed_at = re.findall('\d{4}-\d{2}-\d{2}', requests.get(url, headers=github_headers, timeout=10).json()['pushed_at'])[0] new_pushed_at = re.findall('\d{4}-\d{2}-\d{2}', requests.get(url, headers=github_headers, timeout=10).json()['pushed_at'])[0]
if len(json_str) != 0: if len(json_str) != 0:
tag_name = json_str[0]['tag_name'] tag_name = json_str[0]['tag_name']
if query_pushed_at != new_pushed_at : if query_pushed_at < new_pushed_at :
print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at) print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at)
logging.info("[*] {}工具在数据库里的pushed_at -->{} api的pushed_at -->{}".format(url.split('/')[-1],query_pushed_at,new_pushed_at))
if tag_name != query_tag_name: if tag_name != query_tag_name:
try: try:
update_log = json_str[0]['body'] update_log = json_str[0]['body']
@ -292,7 +263,6 @@ def send_body(url,query_pushed_at,query_tag_name):
conn.commit() conn.commit()
conn.close() conn.close()
print("[+] tools_name -->", tools_name, "pushed_at 已更新，现在pushed_at 为 -->", new_pushed_at,"tag_name 已更新，现在tag_name为 -->",tag_name) print("[+] tools_name -->", tools_name, "pushed_at 已更新，现在pushed_at 为 -->", new_pushed_at,"tag_name 已更新，现在tag_name为 -->",tag_name)
logging.info("[+] tools_name -->{} the tools have releases pushed_at 已更新，现在pushed_at 为 -->{} tag_name 已更新，现在tag_name为 -->{}".format(tools_name,new_pushed_at,tag_name))
elif tag_name == query_tag_name: elif tag_name == query_tag_name:
commits_url = url + "/commits" commits_url = url + "/commits"
commits_url_response_json = requests.get(commits_url).text commits_url_response_json = requests.get(commits_url).text
@ -318,13 +288,11 @@ def send_body(url,query_pushed_at,query_tag_name):
conn.commit() conn.commit()
conn.close() conn.close()
print("[+] tools_name -->",tools_name,"pushed_at 已更新，现在pushed_at 为 -->",new_pushed_at) print("[+] tools_name -->",tools_name,"pushed_at 已更新，现在pushed_at 为 -->",new_pushed_at)
logging.info("[+] tools_name -->{} the tools have releases pushed_at 已更新，现在pushed_at 为 -->{}".format(tools_name,new_pushed_at))
# return update_log, download_url, tools_version # return update_log, download_url, tools_version
else: else:
if query_pushed_at != new_pushed_at: if query_pushed_at != new_pushed_at:
print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at) print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at)
logging.info("[*] {}工具在数据库里的pushed_at -->{} api的pushed_at -->{}".format(url.split('/')[-1],query_pushed_at,new_pushed_at))
json_str = requests.get(url + '/commits', headers=github_headers, timeout=10).json() json_str = requests.get(url + '/commits', headers=github_headers, timeout=10).json()
update_log = json_str[0]['commit']['message'] update_log = json_str[0]['commit']['message']
download_url = json_str[0]['html_url'] download_url = json_str[0]['html_url']
@ -344,7 +312,6 @@ def send_body(url,query_pushed_at,query_tag_name):
conn.commit() conn.commit()
conn.close() conn.close()
print("[+] tools_name -->", tools_name, "pushed_at 已更新，现在pushed_at 为 -->", new_pushed_at) print("[+] tools_name -->", tools_name, "pushed_at 已更新，现在pushed_at 为 -->", new_pushed_at)
logging.info("[+] tools_name -->{} the tools no releases pushed_at 已更新，现在pushed_at 为 -->{}".format(tools_name,new_pushed_at))
# return update_log, download_url # return update_log, download_url
# 创建md5对象 # 创建md5对象
def nmd5(str): def nmd5(str):
@ -410,7 +377,7 @@ def server(text, msg,sckey):
uri = 'https://sc.ftqq.com/{}.send?text={}&desp={}'.format(sckey,text, msg)# 将 xxxx 换成自己的server SCKEY uri = 'https://sc.ftqq.com/{}.send?text={}&desp={}'.format(sckey,text, msg)# 将 xxxx 换成自己的server SCKEY
requests.get(uri, headers=github_headers, timeout=10) requests.get(uri, headers=github_headers, timeout=10)
except Exception as e: except Exception as e:
logging.error("server酱 BUG -> {}".format(e)) pass
# 添加Telegram Bot推送支持 # 添加Telegram Bot推送支持
def tgbot(text, msg,token,group_id): def tgbot(text, msg,token,group_id):
import telegram import telegram
@ -418,7 +385,7 @@ def tgbot(text, msg,token,group_id):
bot = telegram.Bot(token='xxx'.format(token))# Your Telegram Bot Token bot = telegram.Bot(token='xxx'.format(token))# Your Telegram Bot Token
bot.send_message(chat_id=group_id, text='{}\r\n{}'.format(text, msg)) bot.send_message(chat_id=group_id, text='{}\r\n{}'.format(text, msg))
except Exception as e: except Exception as e:
logging.error("tgbot BUG -> {}".format(e)) pass
#判断是否存在该CVE #判断是否存在该CVE
def exist_cve(cve): def exist_cve(cve):
@ -429,7 +396,6 @@ def exist_cve(cve):
des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip() des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip()
return 1 return 1
except Exception as e: except Exception as e:
logging.error("CVE 官网连接 不通 or 不存在该 CVE 具体BUG -> {}".format(e))
return 0 return 0
# 根据cve 名字，获取描述，并翻译 # 根据cve 名字，获取描述，并翻译
@ -442,7 +408,7 @@ def get_cve_des_zh(cve):
des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip() des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip()
return translate(des) return translate(des)
except Exception as e: except Exception as e:
logging.error("CVE 官网连接 不通 具体BUG -> {}".format(e)) pass
#发送CVE信息到钉钉 #发送CVE信息到钉钉
def sendNews(data): def sendNews(data):
try: try:
@ -456,24 +422,19 @@ def sendNews(data):
if load_config()[0] == "dingding": if load_config()[0] == "dingding":
dingding(text, body, load_config()[2], load_config()[3]) dingding(text, body, load_config()[2], load_config()[3])
print("钉钉 发送 CVE 成功") print("钉钉 发送 CVE 成功")
logging.info("钉钉 发送 CVE 成功")
elif load_config()[0] == "server": elif load_config()[0] == "server":
server(text, body, load_config()[2]) server(text, body, load_config()[2])
print("server酱 发送 CVE 成功") print("server酱 发送 CVE 成功")
logging.info("server酱 发送 CVE 成功")
elif load_config()[0] == "tgbot": elif load_config()[0] == "tgbot":
tgbot(text, body, load_config()[2], load_config()[3]) tgbot(text, body, load_config()[2], load_config()[3])
print("tgbot 发送 CVE 成功") print("tgbot 发送 CVE 成功")
logging.info("tgbot 发送 CVE 成功")
except IndexError: except IndexError:
pass pass
except Exception as e: except Exception as e:
print("sendNews 函数 error:{}".format(e)) print("sendNews 函数 error:{}".format(e))
logging.error("sendNews 函数 error:{}".format(e))
#main函数 #main函数
if __name__ == '__main__': if __name__ == '__main__':
print("cve 和 github 发布工具 监控中 ...") print("cve 和 github 发布工具 监控中 ...")
logging.info("cve 和 github 发布工具 监控中 ...")
#初始化部分 #初始化部分
create_database() create_database()
@ -494,8 +455,6 @@ if __name__ == '__main__':
data3 = get_tools_update_list(data2) data3 = get_tools_update_list(data2)
for i in range(len(data3)): for i in range(len(data3)):
try: try:
logging.error("[+++] data3 数据 : api_url - > {} pushed_at - > {} tag_name - > {}".format(data3[i]['api_url'],data3[i]['pushed_at'],data3[i]['tag_name']))
send_body(data3[i]['api_url'],data3[i]['pushed_at'],data3[i]['tag_name']) send_body(data3[i]['api_url'],data3[i]['pushed_at'],data3[i]['tag_name'])
except Exception as e: except Exception as e:
print("main函数 try循环 遇到错误-->{}".format(e)) print("main函数 try循环 遇到错误-->{}".format(e))
logging.error("main函数 try循环 遇到错误-->{}".format(e))