2021-02-25 09:24:45 +08:00
|
|
|
|
#!/usr/bin/python3
|
|
|
|
|
# -*- coding:utf-8 -*-
|
2021-07-30 14:39:00 +08:00
|
|
|
|
# @Author : yhy&ddm&w4ter
|
2021-02-25 09:24:45 +08:00
|
|
|
|
|
2021-09-02 10:30:31 +08:00
|
|
|
|
# 每3分钟检测一次githu
|
|
|
|
|
# 是否有新的cve漏洞提交记录,若有则通过server酱和钉钉机器人推送(二者配置一个即可)
|
2021-02-25 09:24:45 +08:00
|
|
|
|
# 建议使用screen命令运行在自己的linux vps后台上,就可以愉快的接收各种cve了
|
|
|
|
|
|
|
|
|
|
# https://my.oschina.net/u/4581868/blog/4380482
|
|
|
|
|
# https://github.com/kiang70/Github-Monitor
|
2021-07-30 16:35:12 +08:00
|
|
|
|
import json
|
2021-08-25 13:21:00 +08:00
|
|
|
|
from collections import OrderedDict
|
2021-07-30 10:29:21 +08:00
|
|
|
|
import requests, time, re
|
2021-02-25 09:24:45 +08:00
|
|
|
|
import dingtalkchatbot.chatbot as cb
|
|
|
|
|
import datetime
|
2021-07-29 15:11:15 +08:00
|
|
|
|
import hashlib
|
2021-08-25 17:22:48 +08:00
|
|
|
|
import yaml
|
2021-07-29 15:11:15 +08:00
|
|
|
|
from lxml import etree
|
2021-08-25 13:21:00 +08:00
|
|
|
|
import sqlite3
|
2021-08-27 16:54:20 +08:00
|
|
|
|
import logging
|
2021-08-25 17:22:48 +08:00
|
|
|
|
|
2021-09-02 10:30:31 +08:00
|
|
|
|
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.basicConfig(level=logging.DEBUG, filename='run_info.log')
|
|
|
|
|
|
2021-08-25 17:22:48 +08:00
|
|
|
|
#读取配置文件
|
|
|
|
|
def load_config():
|
|
|
|
|
with open('config.yaml', 'r') as f:
|
|
|
|
|
config = yaml.load(f,Loader=yaml.FullLoader)
|
|
|
|
|
github_token = config['all_config']['github_token']
|
2021-08-27 17:30:36 +08:00
|
|
|
|
if int(config['all_config']['dingding'][0]['enable']) == 1:
|
|
|
|
|
dingding_webhook = config['all_config']['dingding'][1]['webhook']
|
|
|
|
|
dingding_secretKey = config['all_config']['dingding'][2]['secretKey']
|
|
|
|
|
app_name = config['all_config']['dingding'][3]['app_name']
|
2021-08-25 17:22:48 +08:00
|
|
|
|
return app_name,github_token,dingding_webhook,dingding_secretKey
|
2021-08-27 17:30:36 +08:00
|
|
|
|
elif int(config['all_config']['server'][0]['enable']) == 1:
|
|
|
|
|
server_sckey = config['all_config']['server'][1]['sckey']
|
|
|
|
|
app_name = config['all_config']['server'][2]['app_name']
|
2021-08-25 17:22:48 +08:00
|
|
|
|
return app_name,github_token,server_sckey
|
2021-08-27 17:30:36 +08:00
|
|
|
|
elif int(config['all_config']['tgbot'][0]['enable']) ==1 :
|
|
|
|
|
tgbot_token = config['all_config']['tgbot'][1]['token']
|
|
|
|
|
tgbot_group_id = config['all_config']['tgbot'][2]['group_id']
|
|
|
|
|
app_name = config['all_config']['tgbot'][3]['app_name']
|
2021-08-25 17:22:48 +08:00
|
|
|
|
return app_name,github_token,tgbot_token,tgbot_group_id
|
2021-08-27 17:30:36 +08:00
|
|
|
|
elif int(config['all_config']['tgbot'][0]['enable']) == 0 and int(config['all_config']['server'][0]['enable']) == 0 and int(config['all_config']['dingding'][0]['enable']) == 0:
|
2021-08-25 17:22:48 +08:00
|
|
|
|
print("[-] 配置文件有误,三个社交软件的enable不能为0")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.error("[-] 配置文件有误,三个社交软件的enable不能为0")
|
2021-08-25 17:22:48 +08:00
|
|
|
|
|
|
|
|
|
github_headers = {
|
|
|
|
|
'Authorization': "token {}".format(load_config()[1]) # 替换自己的github token https://github.com/settings/tokens/new
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-03 16:10:20 +08:00
|
|
|
|
#读取黑名单用户
|
|
|
|
|
def black_user():
|
|
|
|
|
with open('config.yaml', 'r') as f:
|
|
|
|
|
config = yaml.load(f, Loader=yaml.FullLoader)
|
|
|
|
|
black_user = config['all_config']['black_user']
|
|
|
|
|
return black_user
|
2021-08-27 16:54:20 +08:00
|
|
|
|
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#初始化创建数据库
|
|
|
|
|
def create_database():
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
2021-09-02 10:30:31 +08:00
|
|
|
|
# print("[]create_database 函数 连接数据库成功!")
|
|
|
|
|
# logging.info("create_database 函数 连接数据库成功!")
|
2021-08-25 13:21:00 +08:00
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
try:
|
|
|
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS cve_monitor
|
|
|
|
|
(cve_name varchar(255),
|
|
|
|
|
pushed_at varchar(255),
|
|
|
|
|
cve_url varchar(255));''')
|
|
|
|
|
print("成功创建CVE监控表")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info('成功创建CVE监控表')
|
2021-08-25 13:21:00 +08:00
|
|
|
|
cur.execute('''CREATE TABLE IF NOT EXISTS redteam_tools_monitor
|
|
|
|
|
(tools_name varchar(255),
|
|
|
|
|
pushed_at varchar(255),
|
|
|
|
|
tag_name varchar(255));''')
|
|
|
|
|
print("成功创建红队工具监控表")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info('成功创建红队工具监控表')
|
2021-08-25 13:21:00 +08:00
|
|
|
|
except Exception as e:
|
|
|
|
|
print("创建cve监控表失败!报错:{}".format(e))
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.error("创建cve监控表失败!报错:{}".format(e))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
conn.commit() # 数据库存储在硬盘上需要commit 存储在内存中的数据库不需要
|
|
|
|
|
conn.close()
|
2021-08-27 16:54:20 +08:00
|
|
|
|
if load_config()[0] == "dingding":
|
|
|
|
|
dingding("test", "连接成功", load_config()[2], load_config()[3])
|
|
|
|
|
elif load_config()[0] == "server":
|
|
|
|
|
server("test", "连接成功", load_config()[2])
|
|
|
|
|
elif load_config()[0] == "tgbot":
|
|
|
|
|
tgbot("test", "连接成功", load_config()[2], load_config()[3])
|
2021-09-02 10:30:31 +08:00
|
|
|
|
#根据排序获取本年前20条CVE
|
2021-07-29 16:58:48 +08:00
|
|
|
|
def getNews():
|
2021-09-02 10:30:31 +08:00
|
|
|
|
today_cve_info_tmp = []
|
2021-07-29 16:58:48 +08:00
|
|
|
|
try:
|
|
|
|
|
# 抓取本年的
|
|
|
|
|
year = datetime.datetime.now().year
|
|
|
|
|
api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year)
|
2021-07-30 16:35:12 +08:00
|
|
|
|
json_str = requests.get(api, headers=github_headers, timeout=10).json()
|
2021-08-25 13:21:00 +08:00
|
|
|
|
# cve_total_count = json_str['total_count']
|
|
|
|
|
# cve_description = json_str['items'][0]['description']
|
|
|
|
|
today_date = datetime.date.today()
|
|
|
|
|
for i in range(20):
|
|
|
|
|
cve_url = json_str['items'][i]['html_url']
|
2021-09-03 16:10:20 +08:00
|
|
|
|
if cve_url.split("/")[-2] not in black_user():
|
|
|
|
|
try:
|
|
|
|
|
cve_name_tmp = json_str['items'][i]['name'].upper()
|
|
|
|
|
cve_name = re.findall('(CVE\-\d+\-\d+)', cve_name_tmp)[0].upper()
|
|
|
|
|
except Exception as e:
|
|
|
|
|
pass
|
|
|
|
|
pushed_at_tmp = json_str['items'][i]['pushed_at']
|
|
|
|
|
pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
|
|
|
|
|
# today_cve_info_tmp.append({"cve_name": cve_name, "cve_url": cve_url, "pushed_at": pushed_at})
|
|
|
|
|
if pushed_at == str(today_date):
|
|
|
|
|
today_cve_info_tmp.append({"cve_name":cve_name,"cve_url":cve_url,"pushed_at":pushed_at})
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
print("[-] 该{}的更新时间为{},不属于今天的CVE".format(cve_name,pushed_at))
|
|
|
|
|
logging.info("[-] 该{}的更新时间为{},不属于今天的CVE".format(cve_name,pushed_at))
|
2021-08-27 16:54:20 +08:00
|
|
|
|
else:
|
2021-09-03 16:10:20 +08:00
|
|
|
|
logging.info("该{},属于黑名单用户{}提交的CVE".format(cve_name,cve_url.split("/")[-2]))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
today_cve_info = OrderedDict()
|
|
|
|
|
for item in today_cve_info_tmp:
|
|
|
|
|
today_cve_info.setdefault(item['cve_name'], {**item, })
|
|
|
|
|
today_cve_info = list(today_cve_info.values())
|
|
|
|
|
|
|
|
|
|
return today_cve_info
|
|
|
|
|
# return cve_total_count, cve_description, cve_url, cve_name
|
|
|
|
|
#\d{4}-\d{2}-\d{2}
|
2021-07-29 16:58:48 +08:00
|
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
print(e, "github链接不通")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.error(e, "github链接不通")
|
2021-07-29 16:58:48 +08:00
|
|
|
|
return '', '', ''
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#获取到的CVE信息插入到数据库
|
|
|
|
|
def cve_insert_into_sqlite3(data):
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
print("cve_insert_into_sqlite3 函数 打开数据库成功!")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info("cve_insert_into_sqlite3 函数 打开数据库成功!")
|
2021-08-25 13:21:00 +08:00
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
for i in range(len(data)):
|
|
|
|
|
try:
|
2021-09-02 10:30:31 +08:00
|
|
|
|
cve_name = re.findall('(CVE\-\d+\-\d+)', data[i]['cve_name'])[0].upper()
|
2021-08-25 13:21:00 +08:00
|
|
|
|
cur.execute("INSERT INTO cve_monitor (cve_name,pushed_at,cve_url) VALUES ('{}', '{}', '{}')".format(cve_name, data[i]['pushed_at'], data[i]['cve_url']))
|
2021-08-27 16:54:20 +08:00
|
|
|
|
print("cve_insert_into_sqlite3 函数: {}插入数据成功!".format(cve_name))
|
|
|
|
|
logging.info("cve_insert_into_sqlite3 函数: {}插入数据成功!".format(cve_name))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
except Exception as e:
|
|
|
|
|
pass
|
|
|
|
|
conn.commit()
|
|
|
|
|
conn.close()
|
|
|
|
|
#查询数据库里是否存在该CVE的方法
|
|
|
|
|
def query_cve_info_database(cve_name):
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "SELECT cve_name FROM cve_monitor WHERE cve_name = '{}';".format(cve_name)
|
|
|
|
|
cursor = cur.execute(sql_grammar)
|
|
|
|
|
return len(list(cursor))
|
2021-09-02 10:30:31 +08:00
|
|
|
|
#查询数据库里是否存在该tools工具名字的方法
|
|
|
|
|
def query_tools_info_database(tools_name):
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "SELECT tools_name FROM redteam_tools_monitor WHERE tools_name = '{}';".format(tools_name)
|
|
|
|
|
cursor = cur.execute(sql_grammar)
|
|
|
|
|
return len(list(cursor))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#获取不存在数据库里的CVE信息
|
|
|
|
|
def get_today_cve_info(today_cve_info_data):
|
|
|
|
|
today_all_cve_info = []
|
|
|
|
|
# today_cve_info_data = getNews()
|
|
|
|
|
for i in range(len(today_cve_info_data)):
|
2021-09-02 10:30:31 +08:00
|
|
|
|
try:
|
|
|
|
|
today_cve_name = re.findall('(CVE\-\d+\-\d+)', today_cve_info_data[i]['cve_name'])[0].upper()
|
|
|
|
|
logging.info("get_today_cve_info 函数 today_cve_name的值为 - > {}".format(today_cve_name))
|
|
|
|
|
Verify = query_cve_info_database(today_cve_name.upper())
|
|
|
|
|
except Exception as e:
|
|
|
|
|
Verify = 1
|
2021-08-25 13:21:00 +08:00
|
|
|
|
if Verify == 0:
|
|
|
|
|
print("[+] 数据库里不存在{}".format(today_cve_name.upper()))
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info("[+] 数据库里不存在{}".format(today_cve_name.upper()))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
today_all_cve_info.append(today_cve_info_data[i])
|
|
|
|
|
else:
|
|
|
|
|
print("[-] 数据库里存在{}".format(today_cve_name.upper()))
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info("[-] 数据库里存在{}".format(today_cve_name.upper()))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
return today_all_cve_info
|
|
|
|
|
#获取红队工具信息插入到数据库
|
|
|
|
|
def tools_insert_into_sqlite3(data):
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
print("tools_insert_into_sqlite3 函数 打开数据库成功!")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info("tools_insert_into_sqlite3 函数 打开数据库成功!")
|
2021-08-25 13:21:00 +08:00
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
for i in range(len(data)):
|
2021-09-02 10:30:31 +08:00
|
|
|
|
Verify = query_tools_info_database(data[i]['tools_name'])
|
|
|
|
|
if Verify == 0:
|
|
|
|
|
print("[+] 红队工具表数据库里不存在{}".format(data[i]['tools_name']))
|
|
|
|
|
logging.info("[+] 红队工具表数据库里不存在{}".format(data[i]['tools_name']))
|
|
|
|
|
cur.execute("INSERT INTO redteam_tools_monitor (tools_name,pushed_at,tag_name) VALUES ('{}', '{}','{}')".format(data[i]['tools_name'], data[i]['pushed_at'], data[i]['tag_name']))
|
|
|
|
|
print("tools_insert_into_sqlite3 函数: {}插入数据成功!".format(format(data[i]['tools_name'])))
|
|
|
|
|
logging.info("tools_insert_into_sqlite3 函数: {}插入数据成功!".format(format(data[i]['tools_name'])))
|
|
|
|
|
else:
|
|
|
|
|
print("[-] 红队工具表数据库里存在{}".format(data[i]['tools_name']))
|
|
|
|
|
logging.info("[-] 红队工具表数据库里存在{}".format(data[i]['tools_name']))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
conn.commit()
|
|
|
|
|
conn.close()
|
|
|
|
|
#读取本地红队工具链接文件转换成list
|
2021-08-25 17:22:48 +08:00
|
|
|
|
def load_tools_list():
|
|
|
|
|
with open('tools_list.yaml', 'r') as f:
|
|
|
|
|
list = yaml.load(f,Loader=yaml.FullLoader)
|
|
|
|
|
return list['tools_list']
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#获取红队工具的名称,更新时间,版本名称信息
|
2021-07-29 16:58:48 +08:00
|
|
|
|
def get_pushed_at_time(tools_list):
|
2021-08-25 13:21:00 +08:00
|
|
|
|
tools_info_list = []
|
2021-07-29 16:58:48 +08:00
|
|
|
|
for url in tools_list:
|
2021-08-25 13:21:00 +08:00
|
|
|
|
try:
|
2021-09-02 14:29:09 +08:00
|
|
|
|
tools_json = requests.get(url, headers=github_headers, timeout=10).json()
|
|
|
|
|
pushed_at_tmp = tools_json['pushed_at']
|
|
|
|
|
pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0] #获取的是API上的时间
|
|
|
|
|
tools_name = tools_json['name']
|
|
|
|
|
api_url = tools_json['url']
|
|
|
|
|
try:
|
|
|
|
|
releases_json = requests.get(url+"/releases", headers=github_headers, timeout=10).json()
|
|
|
|
|
tag_name = releases_json[0]['tag_name']
|
|
|
|
|
except Exception as e:
|
|
|
|
|
tag_name = "no releases"
|
|
|
|
|
tools_info_list.append({"tools_name":tools_name,"pushed_at":pushed_at,"api_url":api_url,"tag_name":tag_name})
|
2021-08-25 13:21:00 +08:00
|
|
|
|
except Exception as e:
|
2021-09-02 14:29:09 +08:00
|
|
|
|
logging.error("get_pushed_at_time BUG -> {}".format(e))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
|
|
|
|
|
return tools_info_list
|
|
|
|
|
#根据红队名名称查询数据库红队工具的更新时间以及版本名称并返回
|
|
|
|
|
def tools_query_sqlite3(tools_name):
|
2021-09-02 10:30:31 +08:00
|
|
|
|
result_list = []
|
2021-08-25 13:21:00 +08:00
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "SELECT pushed_at,tag_name FROM redteam_tools_monitor WHERE tools_name = '{}';".format(tools_name)
|
|
|
|
|
cursor = cur.execute(sql_grammar)
|
|
|
|
|
for result in cursor:
|
2021-09-02 10:30:31 +08:00
|
|
|
|
result_list.append({"pushed_at":result[0],"tag_name":result[1]})
|
2021-08-25 13:21:00 +08:00
|
|
|
|
conn.close()
|
2021-09-02 10:30:31 +08:00
|
|
|
|
print("[###########] tools_query_sqlite3 函数内 result_list 的值 为 - > {}".format(result_list))
|
|
|
|
|
logging.info("[###########] tools_query_sqlite3 函数内 result_list 的值 为 - > {}".format(result_list))
|
|
|
|
|
return result_list
|
|
|
|
|
#获取更新了的红队工具在数据库里面的时间和版本
|
2021-08-25 13:21:00 +08:00
|
|
|
|
def get_tools_update_list(data):
|
2021-09-02 10:30:31 +08:00
|
|
|
|
tools_update_list = []
|
2021-08-25 13:21:00 +08:00
|
|
|
|
for dist in data:
|
2021-09-02 10:30:31 +08:00
|
|
|
|
print("dist 变量 ->{}".format(dist))
|
|
|
|
|
logging.info("dist 变量 ->{}".format(dist))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
query_result = tools_query_sqlite3(dist['tools_name'])
|
2021-09-02 10:30:31 +08:00
|
|
|
|
today_tools_pushed_at = query_result[0]['pushed_at']
|
|
|
|
|
logging.info("[###########] get_tools_update_list 函数内 today_tools_pushed_at的值 - >{}".format(today_tools_pushed_at))
|
|
|
|
|
# print("[!!] 今日获取时间: ", dist['pushed_at'], "获取数据库时间: ", today_tools_pushed_at, dist['tools_name'])
|
2021-08-25 13:21:00 +08:00
|
|
|
|
if dist['pushed_at'] != today_tools_pushed_at:
|
|
|
|
|
print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name'],"update!!!!")
|
2021-09-02 10:30:31 +08:00
|
|
|
|
logging.info("[!] tools_name: {} 今日API时间: {} 获取数据库时间: {} {}update!!!!".format(dist['tools_name'],dist['pushed_at'],today_tools_pushed_at,dist['tools_name']))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#返回数据库里面的时间和版本
|
2021-09-02 10:30:31 +08:00
|
|
|
|
tools_update_list.append({"api_url":dist['api_url'],"pushed_at":today_tools_pushed_at,"tag_name":query_result[0]['tag_name']})
|
|
|
|
|
else:
|
|
|
|
|
print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name']," no update")
|
|
|
|
|
logging.info("[-] tools_name: {} 今日API时间: {} 获取数据库时间: {} {} no update".format(dist['tools_name'],dist['pushed_at'],today_tools_pushed_at,dist['tools_name']))
|
|
|
|
|
logging.info("get_tools_update_list 函数 tools_update_list数组的值为{}".format(tools_update_list))
|
|
|
|
|
# todo BUG在数组
|
2021-08-25 13:21:00 +08:00
|
|
|
|
return tools_update_list
|
2021-08-25 17:22:48 +08:00
|
|
|
|
#获取更新信息并发送到对应社交软件
|
2021-09-02 14:29:09 +08:00
|
|
|
|
def send_body(url,query_pushed_at,query_tag_name):
|
2021-07-29 16:58:48 +08:00
|
|
|
|
# 考虑到有的工具没有 releases, 则通过 commits 记录获取更新描述
|
|
|
|
|
# 判断是否有 releases 记录
|
2021-07-30 16:35:12 +08:00
|
|
|
|
json_str = requests.get(url + '/releases', headers=github_headers, timeout=10).json()
|
2021-08-25 13:21:00 +08:00
|
|
|
|
new_pushed_at = re.findall('\d{4}-\d{2}-\d{2}', requests.get(url, headers=github_headers, timeout=10).json()['pushed_at'])[0]
|
2021-07-29 16:58:48 +08:00
|
|
|
|
if len(json_str) != 0:
|
2021-08-25 13:21:00 +08:00
|
|
|
|
tag_name = json_str[0]['tag_name']
|
2021-09-02 10:30:31 +08:00
|
|
|
|
if query_pushed_at != new_pushed_at :
|
|
|
|
|
print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at)
|
|
|
|
|
logging.info("[*] {}工具在数据库里的pushed_at -->{} api的pushed_at -->{}".format(url.split('/')[-1],query_pushed_at,new_pushed_at))
|
|
|
|
|
if tag_name != query_tag_name:
|
|
|
|
|
try:
|
|
|
|
|
update_log = json_str[0]['body']
|
|
|
|
|
except Exception as e:
|
|
|
|
|
update_log = "作者未写更新内容"
|
|
|
|
|
download_url = json_str[0]['html_url']
|
|
|
|
|
tools_name = url.split('/')[-1]
|
|
|
|
|
text = r'** ' + tools_name + r' ** 工具,版本更新啦!'
|
|
|
|
|
body = "工具名称:" + tools_name + "\r\n" + "工具地址:" + download_url + "\r\n" + "工具更新日志:" + "\r\n" + update_log
|
|
|
|
|
if load_config()[0] == "dingding":
|
|
|
|
|
dingding(text, body,load_config()[2],load_config()[3])
|
|
|
|
|
elif load_config()[0] == "server":
|
|
|
|
|
server(text, body,load_config()[2])
|
|
|
|
|
elif load_config()[0] == "tgbot":
|
|
|
|
|
tgbot(text,body,load_config()[2],load_config()[3])
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "UPDATE redteam_tools_monitor SET tag_name = '{}' WHERE tools_name='{}'".format(tag_name,tools_name)
|
|
|
|
|
sql_grammar1 = "UPDATE redteam_tools_monitor SET pushed_at = '{}' WHERE tools_name='{}'".format(new_pushed_at, tools_name)
|
|
|
|
|
cur.execute(sql_grammar)
|
|
|
|
|
cur.execute(sql_grammar1)
|
|
|
|
|
conn.commit()
|
|
|
|
|
conn.close()
|
|
|
|
|
print("[+] tools_name -->", tools_name, "pushed_at 已更新,现在pushed_at 为 -->", new_pushed_at,"tag_name 已更新,现在tag_name为 -->",tag_name)
|
|
|
|
|
logging.info("[+] tools_name -->{} the tools have releases pushed_at 已更新,现在pushed_at 为 -->{} tag_name 已更新,现在tag_name为 -->{}".format(tools_name,new_pushed_at,tag_name))
|
|
|
|
|
elif tag_name == query_tag_name:
|
|
|
|
|
commits_url = url + "/commits"
|
|
|
|
|
commits_url_response_json = requests.get(commits_url).text
|
|
|
|
|
commits_json = json.loads(commits_url_response_json)
|
|
|
|
|
tools_name = url.split('/')[-1]
|
|
|
|
|
download_url = commits_json[0]['html_url']
|
|
|
|
|
try:
|
|
|
|
|
update_log = commits_json[0]['commit']['message']
|
|
|
|
|
except Exception as e:
|
|
|
|
|
update_log = "作者未写更新内容,具体点击更新详情地址的URL进行查看"
|
|
|
|
|
text = r'** ' + tools_name + r' ** 工具小更新了一波!'
|
|
|
|
|
body = "工具名称:" + tools_name + "\r\n" + "更新详情地址:" + download_url + "\r\n" + "commit更新日志:" + "\r\n" + update_log
|
|
|
|
|
if load_config()[0] == "dingding":
|
|
|
|
|
dingding(text, body,load_config()[2],load_config()[3])
|
|
|
|
|
elif load_config()[0] == "server":
|
|
|
|
|
server(text, body,load_config()[2])
|
|
|
|
|
elif load_config()[0] == "tgbot":
|
|
|
|
|
tgbot(text,body,load_config()[2],load_config()[3])
|
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "UPDATE redteam_tools_monitor SET pushed_at = '{}' WHERE tools_name='{}'".format(new_pushed_at,tools_name)
|
|
|
|
|
cur.execute(sql_grammar)
|
|
|
|
|
conn.commit()
|
|
|
|
|
conn.close()
|
|
|
|
|
print("[+] tools_name -->",tools_name,"pushed_at 已更新,现在pushed_at 为 -->",new_pushed_at)
|
|
|
|
|
logging.info("[+] tools_name -->{} the tools have releases pushed_at 已更新,现在pushed_at 为 -->{}".format(tools_name,new_pushed_at))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
|
2021-09-02 10:30:31 +08:00
|
|
|
|
# return update_log, download_url, tools_version
|
|
|
|
|
else:
|
|
|
|
|
if query_pushed_at != new_pushed_at:
|
|
|
|
|
print("[*] 数据库里的pushed_at -->", query_pushed_at, ";;;; api的pushed_at -->", new_pushed_at)
|
|
|
|
|
logging.info("[*] {}工具在数据库里的pushed_at -->{} api的pushed_at -->{}".format(url.split('/')[-1],query_pushed_at,new_pushed_at))
|
|
|
|
|
json_str = requests.get(url + '/commits', headers=github_headers, timeout=10).json()
|
|
|
|
|
update_log = json_str[0]['commit']['message']
|
2021-08-25 13:21:00 +08:00
|
|
|
|
download_url = json_str[0]['html_url']
|
|
|
|
|
tools_name = url.split('/')[-1]
|
2021-09-02 10:30:31 +08:00
|
|
|
|
text = r'** ' + tools_name + r' ** 工具更新啦!'
|
|
|
|
|
body = "工具名称:" + tools_name + "\r\n" + "工具地址:" + download_url + "\r\n" + "commit更新日志:" + "\r\n" + update_log
|
2021-08-25 17:22:48 +08:00
|
|
|
|
if load_config()[0] == "dingding":
|
2021-09-02 10:30:31 +08:00
|
|
|
|
dingding(text, body, load_config()[2], load_config()[3])
|
2021-08-25 17:22:48 +08:00
|
|
|
|
elif load_config()[0] == "server":
|
2021-09-02 10:30:31 +08:00
|
|
|
|
server(text, body, load_config()[2])
|
2021-08-25 17:22:48 +08:00
|
|
|
|
elif load_config()[0] == "tgbot":
|
2021-09-02 10:30:31 +08:00
|
|
|
|
tgbot(text, body, load_config()[2], load_config()[3])
|
2021-08-25 13:21:00 +08:00
|
|
|
|
conn = sqlite3.connect('data.db')
|
|
|
|
|
cur = conn.cursor()
|
|
|
|
|
sql_grammar = "UPDATE redteam_tools_monitor SET pushed_at = '{}' WHERE tools_name='{}'".format(new_pushed_at,tools_name)
|
|
|
|
|
cur.execute(sql_grammar)
|
|
|
|
|
conn.commit()
|
|
|
|
|
conn.close()
|
2021-09-02 10:30:31 +08:00
|
|
|
|
print("[+] tools_name -->", tools_name, "pushed_at 已更新,现在pushed_at 为 -->", new_pushed_at)
|
|
|
|
|
logging.info("[+] tools_name -->{} the tools no releases pushed_at 已更新,现在pushed_at 为 -->{}".format(tools_name,new_pushed_at))
|
|
|
|
|
# return update_log, download_url
|
2021-07-29 15:11:15 +08:00
|
|
|
|
# 创建md5对象
|
|
|
|
|
def nmd5(str):
|
|
|
|
|
m = hashlib.md5()
|
|
|
|
|
b = str.encode(encoding='utf-8')
|
|
|
|
|
m.update(b)
|
|
|
|
|
str_md5 = m.hexdigest()
|
2021-07-30 16:35:12 +08:00
|
|
|
|
return str_md5
|
2021-07-29 15:11:15 +08:00
|
|
|
|
# 有道翻译
|
|
|
|
|
def translate(word):
|
|
|
|
|
headerstr = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
|
|
|
|
|
bv = nmd5(headerstr)
|
2021-07-30 16:35:12 +08:00
|
|
|
|
lts = str(round(time.time() * 1000))
|
2021-07-29 15:11:15 +08:00
|
|
|
|
salt = lts + '90'
|
|
|
|
|
# 如果翻译失败,{'errorCode': 50} 请查看 fanyi.min.js: https://shared.ydstatic.com/fanyi/newweb/v1.1.7/scripts/newweb/fanyi.min.js
|
|
|
|
|
# 搜索 fanyideskweb sign: n.md5("fanyideskweb" + e + i + "Y2FYu%TNSbMCxc3t2u^XT") ,Y2FYu%TNSbMCxc3t2u^XT是否改变,替换即可
|
2021-07-30 16:35:12 +08:00
|
|
|
|
strexample = 'fanyideskweb' + word + salt + 'Y2FYu%TNSbMCxc3t2u^XT'
|
2021-07-29 15:11:15 +08:00
|
|
|
|
sign = nmd5(strexample)
|
|
|
|
|
data = {
|
|
|
|
|
'i': word,
|
|
|
|
|
'from': 'AUTO',
|
|
|
|
|
'to': 'AUTO',
|
|
|
|
|
'smartresult': 'dict',
|
|
|
|
|
'client': 'fanyideskweb',
|
2021-07-30 16:35:12 +08:00
|
|
|
|
'salt': salt,
|
|
|
|
|
'sign': sign,
|
2021-07-29 15:11:15 +08:00
|
|
|
|
'lts': lts,
|
|
|
|
|
'bv': bv,
|
|
|
|
|
'doctype': 'json',
|
|
|
|
|
'version': '2.1',
|
|
|
|
|
'keyfrom': 'fanyi.web',
|
|
|
|
|
'action': 'FY_BY_CLICKBUTTION',
|
|
|
|
|
}
|
2021-07-30 16:35:12 +08:00
|
|
|
|
url = 'http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule'
|
|
|
|
|
header = {
|
|
|
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36',
|
|
|
|
|
'Referer': 'http://fanyi.youdao.com/',
|
2021-07-29 15:11:15 +08:00
|
|
|
|
'Origin': 'http://fanyi.youdao.com',
|
2021-07-30 16:35:12 +08:00
|
|
|
|
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
|
|
|
|
'X-Requested-With': 'XMLHttpRequest',
|
|
|
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
|
|
'Accept-Language': 'zh-CN,zh;q=0.9',
|
2021-07-29 15:11:15 +08:00
|
|
|
|
'Connection': 'keep-alive',
|
|
|
|
|
'Host': 'fanyi.youdao.com',
|
2021-07-30 16:35:12 +08:00
|
|
|
|
'cookie': '_ntes_nnid=937f1c788f1e087cf91d616319dc536a,1564395185984; OUTFOX_SEARCH_USER_ID_NCOO=; OUTFOX_SEARCH_USER_ID=-10218418@11.136.67.24; JSESSIONID=; ___rl__test__cookies=1'
|
2021-07-29 15:11:15 +08:00
|
|
|
|
}
|
|
|
|
|
res = requests.post(url=url, data=data, headers=header)
|
2021-07-29 16:58:48 +08:00
|
|
|
|
result_dict = res.json()
|
2021-07-29 15:11:15 +08:00
|
|
|
|
result = ""
|
|
|
|
|
for json_str in result_dict['translateResult'][0]:
|
|
|
|
|
tgt = json_str['tgt']
|
|
|
|
|
result += tgt
|
|
|
|
|
return result
|
2021-08-25 17:22:48 +08:00
|
|
|
|
|
2021-02-25 09:24:45 +08:00
|
|
|
|
# 钉钉
|
2021-08-25 17:22:48 +08:00
|
|
|
|
def dingding(text, msg,webhook,secretKey):
|
2021-07-30 16:35:12 +08:00
|
|
|
|
ding = cb.DingtalkChatbot(webhook, secret=secretKey)
|
|
|
|
|
ding.send_text(msg='{}\r\n{}'.format(text, msg), is_at_all=False)
|
2021-02-25 09:24:45 +08:00
|
|
|
|
# server酱 http://sc.ftqq.com/?c=code
|
2021-08-25 17:22:48 +08:00
|
|
|
|
def server(text, msg,sckey):
|
2021-09-02 14:29:09 +08:00
|
|
|
|
try:
|
|
|
|
|
uri = 'https://sc.ftqq.com/{}.send?text={}&desp={}'.format(sckey,text, msg)# 将 xxxx 换成自己的server SCKEY
|
|
|
|
|
requests.get(uri, headers=github_headers, timeout=10)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logging.error("server酱 BUG -> {}".format(e))
|
2021-07-03 18:27:24 +08:00
|
|
|
|
# 添加Telegram Bot推送支持
|
2021-08-25 17:22:48 +08:00
|
|
|
|
def tgbot(text, msg,token,group_id):
|
2021-07-03 18:27:24 +08:00
|
|
|
|
import telegram
|
2021-09-02 14:29:09 +08:00
|
|
|
|
try:
|
|
|
|
|
bot = telegram.Bot(token='xxx'.format(token))# Your Telegram Bot Token
|
|
|
|
|
bot.send_message(chat_id=group_id, text='{}\r\n{}'.format(text, msg))
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logging.error("tgbot BUG -> {}".format(e))
|
2021-07-29 15:11:15 +08:00
|
|
|
|
# 根据cve 名字,获取描述,并翻译
|
|
|
|
|
def get_cve_des_zh(cve):
|
2021-08-25 13:21:00 +08:00
|
|
|
|
time.sleep(3)
|
2021-09-02 14:29:09 +08:00
|
|
|
|
try:
|
|
|
|
|
query_cve_url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + cve
|
|
|
|
|
response = requests.get(query_cve_url, headers=github_headers, timeout=10)
|
|
|
|
|
html = etree.HTML(response.text)
|
|
|
|
|
des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip()
|
|
|
|
|
return translate(des)
|
|
|
|
|
except Exception as e:
|
|
|
|
|
logging.error("CVE 官网连接 不通 具体BUG -> {}".format(e))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#发送CVE信息到钉钉
|
|
|
|
|
def sendNews(data):
|
|
|
|
|
try:
|
|
|
|
|
text = r'有新的CVE送达!'
|
|
|
|
|
# 获取 cve 名字 ,根据cve 名字,获取描述,并翻译
|
|
|
|
|
for i in range(len(data)):
|
|
|
|
|
try:
|
2021-09-02 10:30:31 +08:00
|
|
|
|
cve_name = re.findall('(CVE\-\d+\-\d+)', data[i]['cve_name'])[0].upper()
|
2021-07-30 16:35:12 +08:00
|
|
|
|
cve_zh = get_cve_des_zh(cve_name)
|
2021-08-25 17:22:48 +08:00
|
|
|
|
body = "CVE编号:" + cve_name + "\r\n" + "Github地址:" + str(data[i]['cve_url']) + "\r\n" + "CVE描述:" + "\r\n" + cve_zh
|
|
|
|
|
if load_config()[0] == "dingding":
|
|
|
|
|
dingding(text, body, load_config()[2], load_config()[3])
|
2021-08-27 16:54:20 +08:00
|
|
|
|
print("钉钉 发送 CVE 成功")
|
|
|
|
|
logging.info("钉钉 发送 CVE 成功")
|
2021-08-25 17:22:48 +08:00
|
|
|
|
elif load_config()[0] == "server":
|
|
|
|
|
server(text, body, load_config()[2])
|
2021-08-27 16:54:20 +08:00
|
|
|
|
print("server酱 发送 CVE 成功")
|
|
|
|
|
logging.info("server酱 发送 CVE 成功")
|
2021-08-25 17:22:48 +08:00
|
|
|
|
elif load_config()[0] == "tgbot":
|
|
|
|
|
tgbot(text, body, load_config()[2], load_config()[3])
|
2021-08-27 16:54:20 +08:00
|
|
|
|
print("tgbot 发送 CVE 成功")
|
|
|
|
|
logging.info("tgbot 发送 CVE 成功")
|
2021-08-25 13:21:00 +08:00
|
|
|
|
except IndexError:
|
|
|
|
|
pass
|
|
|
|
|
except Exception as e:
|
2021-08-27 16:54:20 +08:00
|
|
|
|
print("sendNews 函数 error:{}".format(e))
|
|
|
|
|
logging.error("sendNews 函数 error:{}".format(e))
|
2021-08-25 13:21:00 +08:00
|
|
|
|
#main函数
|
2021-02-25 09:24:45 +08:00
|
|
|
|
if __name__ == '__main__':
|
2021-08-25 13:21:00 +08:00
|
|
|
|
print("cve 和 github 发布工具 监控中 ...")
|
2021-08-27 16:54:20 +08:00
|
|
|
|
logging.info("cve 和 github 发布工具 监控中 ...")
|
2021-08-25 13:21:00 +08:00
|
|
|
|
|
2021-08-27 16:54:20 +08:00
|
|
|
|
#初始化部分
|
|
|
|
|
create_database()
|
|
|
|
|
|
|
|
|
|
while True:
|
2021-09-02 10:30:31 +08:00
|
|
|
|
tools_list = load_tools_list()
|
|
|
|
|
tools_data = get_pushed_at_time(tools_list)
|
|
|
|
|
tools_insert_into_sqlite3(tools_data)
|
|
|
|
|
#CVE部分
|
|
|
|
|
cve_data = getNews()
|
|
|
|
|
today_cve_data = get_today_cve_info(cve_data)
|
|
|
|
|
sendNews(today_cve_data)
|
|
|
|
|
cve_insert_into_sqlite3(today_cve_data)
|
|
|
|
|
#红队工具部分
|
|
|
|
|
time.sleep(3)
|
|
|
|
|
tools_list_new = load_tools_list()
|
|
|
|
|
data2 = get_pushed_at_time(tools_list_new)
|
|
|
|
|
data3 = get_tools_update_list(data2)
|
|
|
|
|
for i in range(len(data3)):
|
|
|
|
|
try:
|
|
|
|
|
logging.error("[+++] data3 数据 : api_url - > {} pushed_at - > {} tag_name - > {}".format(data3[i]['api_url'],data3[i]['pushed_at'],data3[i]['tag_name']))
|
2021-09-02 14:29:09 +08:00
|
|
|
|
send_body(data3[i]['api_url'],data3[i]['pushed_at'],data3[i]['tag_name'])
|
2021-09-02 10:30:31 +08:00
|
|
|
|
except Exception as e:
|
|
|
|
|
print("main函数 try循环 遇到错误-->{}".format(e))
|
|
|
|
|
logging.error("main函数 try循环 遇到错误-->{}".format(e))
|