diff --git a/README.md b/README.md index 9e75612..26b276e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## 更新 -合并@[Xc1Ym](https://github.com/yhy0/github-cve-monitor/pull/6) +requests 请求 去除正则，使用json ; 增加监控github发布的工具更新情况 通过https://cve.mitre.org/ 获取CVE具体描述, 并通过有道翻译为中文 diff --git a/github_cve_monitor.py b/github_cve_monitor.py index 7079b84..17bc835 100644 --- a/github_cve_monitor.py +++ b/github_cve_monitor.py @@ -8,14 +8,53 @@ # https://my.oschina.net/u/4581868/blog/4380482 # https://github.com/kiang70/Github-Monitor -import requests, re, time +import requests, time import dingtalkchatbot.chatbot as cb import datetime import hashlib -import json from lxml import etree +# 抓取本年cve +def getNews(): + try: + # 抓取本年的 + year = datetime.datetime.now().year + api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year) + json_str = requests.get(api).json() + cve_total_count = json_str['total_count'] + cve_description = json_str['items'][0]['description'] + cve_url = json_str['items'][0]['html_url'] + return cve_total_count, cve_description, cve_url + + except Exception as e: + print(e, "github链接不通") + return '', '', '' + +# 通过 pushed_at 检查工具是否更新 +def get_pushed_at_time(tools_list): + total_list = [] + for url in tools_list: + pushed_at = requests.get(url).json()['pushed_at'] + total_list.append(pushed_at) + + return total_list + +def get_update_log(url): + # 考虑到有的工具没有 releases, 则通过 commits 记录获取更新描述 + + # 判断是否有 releases 记录 + json_str = requests.get(url + '/releases').json() + if len(json_str) != 0: + update_log = json_str[0]['body'] + download_url = json_str[0]['html_url'] + return update_log, download_url + else: + json_str = requests.get(url + '/releases').json() + update_log = json_str[0]['commit']['message'] + return update_log, '' + + # 创建md5对象 def nmd5(str): m = hashlib.md5() @@ -68,10 +107,7 @@ def translate(word): } res = requests.post(url=url, data=data, headers=header) - t = res.content.decode('utf8') - - # 把返回来的json字符串解析成字典 - result_dict = json.loads(t) + result_dict = res.json() result = "" for json_str in result_dict['translateResult'][0]: @@ -80,22 +116,6 @@ def translate(word): return result -def getNews(): - try: - # 抓取本年的 - year = datetime.datetime.now().year - api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year) - req = requests.get(api).text - cve_total_count = re.findall('"total_count":*.{1,10}"incomplete_results"',req)[0][14:17] - cve_description = re.findall('"description":*.{1,200}"fork"',req)[0].replace("\",\"fork\"",'').replace("\"description\":\"",'') - cve_url = re.findall('"svn_url":*.{1,200}"homepage"',req)[0].replace("\",\"homepage\"",'').replace("\"svn_url\":\"",'') - # 不推送 fork - if cve_description != '"description":null,"fork"': - return cve_total_count, cve_description, cve_url - - except Exception as e: - print (e, "github链接不通") - # 钉钉 def dingding(text, msg): # 将此处换为钉钉机器人的api @@ -120,18 +140,13 @@ def tgbot(text,msg): # 通过检查name 和 description 中是否存在test字样，排除test def regular(req): - cve_name = re.findall('"name":*.{1,200}"full_name"', req)[0].replace("\"name\":\"",'').replace("\",\"full_name\"",'') - cve_description = re.findall('"description":*.{1,200}"fork"', req)[0].replace("\",\"fork\"", '').replace( - "\"description\":\"", '') - + cve_name = req['items'][0]['name'] + cve_description = req['items'][0]['description'] if cve_name.lower().find('test') == -1 and cve_description.lower().find('test') == -1: return True return False -# 获取cve 名字 ， -def get_cve_name(req): - cve_name = re.findall('"name":*.{1,200}"full_name"', req)[0].replace("\"name\":\"", '').replace("\",\"full_name\"",'') - return cve_name + # 根据cve 名字，获取描述，并翻译 def get_cve_des_zh(cve): query_cve_url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + cve @@ -140,29 +155,49 @@ def get_cve_des_zh(cve): des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip() return translate(des) -def sendNews(): +def sendNews(tools_list): while True: try: - print("cve 监控中 ...") - # 抓取本年的 + print("cve 和 github 发布工具 监控中 ...") + # 抓取本年的cve year = datetime.datetime.now().year api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year) # 请求API - req = requests.get(api).text - # 正则获取 - total_count = re.findall('"total_count":*.{1,10}"incomplete_results"', req)[0][14:17] + req = requests.get(api).json() + total_count = req['total_count'] + + # 通过 pushed_at 检查工具是否更新 + time_list1 = get_pushed_at_time(tools_list) # 监控时间间隔3分钟 time.sleep(180) - # 推送正文内容 - # 推送标题 - text = r'有新的CVE送达！' - regular(req) + + time_list2 = get_pushed_at_time(tools_list) + + for i in range(len(tools_list)): + # 两次时间不相等，则代表工具更新 + if time_list1[i] != time_list2[i]: + update_log, download_url = get_update_log(tools_list[i]) + tools_name = tools_list[i].split('/')[-1] + text = r'** ' + tools_name + r' ** 工具更新啦!' + # body = '' + if download_url != '': + body = "工具名称：" + tools_name + "\r\n" + "工具下载地址：" + download_url + "\r\n" + "工具更新日志：" + update_log + else: + body = "工具名称：" + "\r\n" + "工具更新日志：" + update_log + # 三选一即可，没配置的 注释或者删掉 + # server(text, body) + dingding(text, body) + # tgbot(text,body) + print(body) # 检查name 和 description 中是否存在test字样 和 是否更新 if regular(req) and total_count != getNews()[0]: - cve = get_cve_name(req) - cve_zh = get_cve_des_zh(cve) + # 推送正文内容 + # 推送标题 + text = r'有新的CVE送达！' + # 获取 cve 名字 ，根据cve 名字，获取描述，并翻译 + cve_zh = get_cve_des_zh(req['items'][0]['name']) msg = "CVE编号："+ str(getNews()[1]) + "\r\n"+"Github地址："+ str(getNews()[2]) + "\r\n" + "CVE描述："+ cve_zh # 三选一即可，没配置的 注释或者删掉 @@ -170,12 +205,24 @@ def sendNews(): dingding(text, msg) # tgbot(text,msg) print(msg) - else: - pass except Exception as e: - raise e + print("Program runing error:{}".format(e)) if __name__ == '__main__': - sendNews() + tools_list = [ + "https://api.github.com/repos/BeichenDream/Godzilla", + "https://api.github.com/repos/rebeyond/Behinder", + "https://api.github.com/repos/AntSwordProject/antSword", + "https://api.github.com/repos/j1anFen/shiro_attack", + "https://api.github.com/repos/yhy0/ExpDemo-JavaFX", + "https://api.github.com/repos/yhy0/github-cve-monitor", + "https://api.github.com/repos/gentilkiwi/mimikatz", + "https://api.github.com/repos/ehang-io/nps", + "https://api.github.com/repos/chaitin/xray", + "https://api.github.com/repos/FunnyWolf/pystinger", + "https://api.github.com/repos/L-codes/Neo-reGeorg", + "https://api.github.com/repos/shadow1ng/fscan", + ] + sendNews(tools_list)