From f97b8ba92ec7a9cea02801b71609191ac4e946a4 Mon Sep 17 00:00:00 2001 From: yhy <31311038+yhy0@users.noreply.github.com> Date: Sat, 8 Oct 2022 17:39:57 +0800 Subject: [PATCH] =?UTF-8?q?tools=5Flist=20=E6=96=B0=E5=A2=9E=E7=94=A8?= =?UTF-8?q?=E6=88=B7=E4=BB=93=E5=BA=93=E7=9B=91=E6=8E=A7,=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E4=B8=80=E4=BA=9B=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- github_cve_monitor.py | 81 ++++++++++++++++++++++++++++++++++++------- tools_list.yaml | 14 +++++--- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/github_cve_monitor.py b/github_cve_monitor.py index 182df17..f612a1d 100644 --- a/github_cve_monitor.py +++ b/github_cve_monitor.py @@ -49,7 +49,7 @@ def load_config(): app_name = config['all_config']['tgbot'][3]['app_name'] return app_name,github_token,tgbot_token,tgbot_group_id, translate elif int(config['all_config']['tgbot'][0]['enable']) == 0 and int(config['all_config']['feishu'][0]['enable']) == 0 and int(config['all_config']['server'][0]['enable']) == 0 and int(config['all_config']['pushplus'][0]['enable']) == 0 and int(config['all_config']['dingding'][0]['enable']) == 0: - print("[-] 配置文件有误，五个社交软件的enable不能为0") + print("[-] 配置文件有误, 五个社交软件的enable不能为0") github_headers = { 'Authorization': "token {}".format(load_config()[1]) @@ -84,6 +84,9 @@ def create_database(): pushed_at varchar(255), tag_name varchar(255));''') print("成功创建红队工具监控表") + cur.execute('''CREATE TABLE IF NOT EXISTS user_monitor + (repo_name varchar(255));''') + print("成功创建大佬仓库监控表") except Exception as e: print("创建监控表失败！报错：{}".format(e)) conn.commit() # 数据库存储在硬盘上需要commit 存储在内存中的数据库不需要 @@ -116,12 +119,12 @@ def getNews(): try: cve_name_tmp = json_str['items'][i]['name'].upper() cve_name = re.findall('(CVE\-\d+\-\d+)', cve_name_tmp)[0].upper() - pushed_at_tmp = json_str['items'][i]['pushed_at'] + pushed_at_tmp = json_str['items'][i]['created_at'] pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0] if pushed_at == str(today_date): today_cve_info_tmp.append({"cve_name": cve_name, "cve_url": cve_url, "pushed_at": pushed_at}) else: - print("[-] 该{}的更新时间为{}，不属于今天的CVE".format(cve_name, pushed_at)) + print("[-] 该{}的更新时间为{}, 不属于今天的CVE".format(cve_name, pushed_at)) except Exception as e: pass else: @@ -154,7 +157,7 @@ def getKeywordNews(keyword): if keyword_url.split("/")[-2] not in black_user(): try: keyword_name = json_str['items'][i]['name'] - pushed_at_tmp = json_str['items'][i]['pushed_at'] + pushed_at_tmp = json_str['items'][i]['created_at'] pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0] if pushed_at == str(today_date): today_keyword_info_tmp.append({"keyword_name": keyword_name, "keyword_url": keyword_url, "pushed_at": pushed_at}) @@ -285,7 +288,7 @@ def tools_insert_into_sqlite3(data): def load_tools_list(): with open('tools_list.yaml', 'r') as f: list = yaml.load(f,Loader=yaml.FullLoader) - return list['tools_list'], list['keyword_list'] + return list['tools_list'], list['keyword_list'], list['user_list'] #获取红队工具的名称，更新时间，版本名称信息 def get_pushed_at_time(tools_list): tools_info_list = [] @@ -303,7 +306,7 @@ def get_pushed_at_time(tools_list): tag_name = "no releases" tools_info_list.append({"tools_name":tools_name,"pushed_at":pushed_at,"api_url":api_url,"tag_name":tag_name}) except Exception as e: - print(e) + print("get_pushed_at_time ", e) pass return tools_info_list @@ -335,6 +338,56 @@ def get_tools_update_list(data): else: print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name']," no update") return tools_update_list + + +# 监控用户是否新增仓库，不是 fork 的 +def getUserRepos(user): + try: + api = "https://api.github.com/users/{}/repos".format(user) + json_str = requests.get(api, headers=github_headers, timeout=10).json() + today_date = datetime.date.today() + + for i in range(0, len(json_str)): + created_at = re.findall('\d{4}-\d{2}-\d{2}', json_str[i]['created_at'])[0] + if json_str[i]['fork'] == False and created_at == str(today_date): + Verify = user_insert_into_sqlite3(json_str[i]['full_name']) + print(json_str[i]['full_name'], Verify) + if Verify == 0: + name = json_str[i]['name'] + try: + description = json_str[i]['description'] + except Exception as e: + description = "作者未写描述" + download_url = json_str[i]['html_url'] + text = r'大佬' + r'** ' + user + r' ** ' + r'又分享了一款工具! ' + body = "工具名称: " + name + " \r\n" + "工具地址: " + download_url + " \r\n" + "工具描述: " + "" + description + if load_config()[0] == "dingding": + dingding(text, body,load_config()[2],load_config()[3]) + if load_config()[0] == "server": + server(text, body,load_config()[2]) + if load_config()[0] == "pushplus": + pushplus(text, body,load_config()[2]) + if load_config()[0] == "tgbot": + tgbot(text,body,load_config()[2],load_config()[3]) + except Exception as e: + print(e, "github链接不通") + +#获取用户或者组织信息插入到数据库 +def user_insert_into_sqlite3(repo_name): + conn = sqlite3.connect('data.db') + cur = conn.cursor() + sql_grammar = "SELECT repo_name FROM user_monitor WHERE repo_name = '{}';".format(repo_name) + Verify = len(list(cur.execute(sql_grammar))) + if Verify == 0: + print("[+] 用户仓库表数据库里不存在{}".format(repo_name)) + cur.execute("INSERT INTO user_monitor (repo_name) VALUES ('{}')".format(repo_name)) + print("user_insert_into_sqlite3 函数: {}插入数据成功！".format(repo_name)) + else: + print("[-] 用户仓库表数据库里存在{}".format(repo_name)) + conn.commit() + conn.close() + return Verify + #获取更新信息并发送到对应社交软件 def send_body(url,query_pushed_at,query_tag_name): # 考虑到有的工具没有 releases, 则通过 commits 记录获取更新描述 @@ -533,7 +586,7 @@ def get_cve_des_zh(cve): response = requests.get(query_cve_url, headers=github_headers, timeout=10) html = etree.HTML(response.text) des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip() - if load_config()[4]: + if load_config()[-1]: return translate(des) return des except Exception as e: @@ -599,14 +652,18 @@ def sendKeywordNews(keyword, data): #main函数 if __name__ == '__main__': - print("cve 和 github 发布工具 监控中 ...") + print("cve 、github 工具 和 大佬仓库 监控中 ...") #初始化部分 create_database() while True: - tools_list, keyword_list = load_tools_list() + tools_list, keyword_list, user_list = load_tools_list() tools_data = get_pushed_at_time(tools_list) tools_insert_into_sqlite3(tools_data) # 获取文件中的工具列表，并从 github 获取相关信息，存储下来 + + print("\r\n\t\t 用户仓库监控 \t\t\r\n") + for user in user_list: + getUserRepos(user) #CVE部分 print("\r\n\t\t CVE 监控 \t\t\r\n") cve_data = getNews() @@ -618,6 +675,7 @@ if __name__ == '__main__': print("\r\n\t\t 关键字监控 \t\t\r\n") # 关键字监控 , 最好不要太多关键字，防止 github 次要速率限制 https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits= for keyword in keyword_list: + time.sleep(1) # 每个关键字停 1s ，防止关键字过多导致速率限制 keyword_data = getKeywordNews(keyword) if len(keyword_data) > 0: @@ -627,9 +685,8 @@ if __name__ == '__main__': keyword_insert_into_sqlite3(today_keyword_data) print("\r\n\t\t 红队工具监控 \t\t\r\n") - # 红队工具部分 - time.sleep(3*60) - tools_list_new, keyword_list = load_tools_list() + time.sleep(5*60) + tools_list_new, keyword_list, user_list = load_tools_list() data2 = get_pushed_at_time(tools_list_new) # 再次从文件中获取工具列表，并从 github 获取相关信息, data3 = get_tools_update_list(data2) # 与 3 分钟前数据进行对比，如果在三分钟内有新增工具清单或者工具有更新则通知一下用户 for i in range(len(data3)): diff --git a/tools_list.yaml b/tools_list.yaml index b818611..37962f4 100644 --- a/tools_list.yaml +++ b/tools_list.yaml @@ -3,7 +3,6 @@ tools_list: - https://api.github.com/repos/rebeyond/Behinder - https://api.github.com/repos/AntSwordProject/antSword - https://api.github.com/repos/j1anFen/shiro_attack - - https://api.github.com/repos/yhy0/ExpDemo-JavaFX - https://api.github.com/repos/yhy0/github-cve-monitor - https://api.github.com/repos/gentilkiwi/mimikatz - https://api.github.com/repos/ehang-io/nps @@ -16,8 +15,15 @@ tools_list: keyword_list: - Sql注入 - - rce - cnvd - 未授权 - - 注入 - - 命令执行 \ No newline at end of file + +user_list: + - yhy0 + - su18 + - BeichenDream + - phith0n + - zhzyker + - lijiejie + - projectdiscovery + - HavocFramework \ No newline at end of file