tools_list 新增用户仓库监控,修复一些问题

github_cve_monitor.py

@@ -49,7 +49,7 @@ def load_config():
             app_name = config['all_config']['tgbot'][3]['app_name']
             return app_name,github_token,tgbot_token,tgbot_group_id, translate
         elif int(config['all_config']['tgbot'][0]['enable']) == 0 and int(config['all_config']['feishu'][0]['enable']) == 0 and int(config['all_config']['server'][0]['enable']) == 0 and int(config['all_config']['pushplus'][0]['enable']) == 0 and int(config['all_config']['dingding'][0]['enable']) == 0:
-            print("[-] 配置文件有误，五个社交软件的enable不能为0")
+            print("[-] 配置文件有误, 五个社交软件的enable不能为0")
 
 github_headers = {
     'Authorization': "token {}".format(load_config()[1])
@@ -84,6 +84,9 @@ def create_database():
                     pushed_at varchar(255),
                     tag_name varchar(255));''')
         print("成功创建红队工具监控表")
+        cur.execute('''CREATE TABLE IF NOT EXISTS user_monitor
+                   (repo_name varchar(255));''')
+        print("成功创建大佬仓库监控表")
     except Exception as e:
         print("创建监控表失败！报错：{}".format(e))
     conn.commit()  # 数据库存储在硬盘上需要commit  存储在内存中的数据库不需要
@@ -116,12 +119,12 @@ def getNews():
                 try:
                     cve_name_tmp = json_str['items'][i]['name'].upper()
                     cve_name = re.findall('(CVE\-\d+\-\d+)', cve_name_tmp)[0].upper()
-                    pushed_at_tmp = json_str['items'][i]['pushed_at']
+                    pushed_at_tmp = json_str['items'][i]['created_at']
                     pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
                     if pushed_at == str(today_date):
                         today_cve_info_tmp.append({"cve_name": cve_name, "cve_url": cve_url, "pushed_at": pushed_at})
                     else:
-                        print("[-] 该{}的更新时间为{}，不属于今天的CVE".format(cve_name, pushed_at))
+                        print("[-] 该{}的更新时间为{}, 不属于今天的CVE".format(cve_name, pushed_at))
                 except Exception as e:
                     pass
             else:
@@ -154,7 +157,7 @@ def getKeywordNews(keyword):
             if keyword_url.split("/")[-2] not in black_user():
                 try:
                     keyword_name = json_str['items'][i]['name']
-                    pushed_at_tmp = json_str['items'][i]['pushed_at']
+                    pushed_at_tmp = json_str['items'][i]['created_at']
                     pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
                     if pushed_at == str(today_date):
                         today_keyword_info_tmp.append({"keyword_name": keyword_name, "keyword_url": keyword_url, "pushed_at": pushed_at})
@@ -285,7 +288,7 @@ def tools_insert_into_sqlite3(data):
 def load_tools_list():
     with open('tools_list.yaml', 'r') as f:
         list = yaml.load(f,Loader=yaml.FullLoader)
-        return list['tools_list'], list['keyword_list']
+        return list['tools_list'], list['keyword_list'], list['user_list']
 #获取红队工具的名称，更新时间，版本名称信息
 def get_pushed_at_time(tools_list):
     tools_info_list = []
@@ -303,7 +306,7 @@ def get_pushed_at_time(tools_list):
                 tag_name = "no releases"
             tools_info_list.append({"tools_name":tools_name,"pushed_at":pushed_at,"api_url":api_url,"tag_name":tag_name})
         except Exception as e:
-            print(e)
+            print("get_pushed_at_time ", e)
             pass
 
     return tools_info_list
@@ -335,6 +338,56 @@ def get_tools_update_list(data):
             else:
                 print("今日获取时间: ",dist['pushed_at'],"获取数据库时间: ",today_tools_pushed_at,dist['tools_name'],"   no update")
     return tools_update_list
+
+
+# 监控用户是否新增仓库，不是 fork 的
+def getUserRepos(user):
+    try:
+        api = "https://api.github.com/users/{}/repos".format(user)
+        json_str = requests.get(api, headers=github_headers, timeout=10).json()
+        today_date = datetime.date.today()
+
+        for i in range(0, len(json_str)):
+            created_at = re.findall('\d{4}-\d{2}-\d{2}', json_str[i]['created_at'])[0]
+            if json_str[i]['fork'] == False and created_at == str(today_date):
+                Verify = user_insert_into_sqlite3(json_str[i]['full_name'])
+                print(json_str[i]['full_name'], Verify)
+                if Verify == 0:
+                    name = json_str[i]['name']
+                    try:
+                        description = json_str[i]['description']
+                    except Exception as e:
+                        description = "作者未写描述"
+                    download_url = json_str[i]['html_url']
+                    text = r'大佬' + r'** ' + user + r' ** ' + r'又分享了一款工具! '
+                    body = "工具名称: " + name + " \r\n" + "工具地址: " + download_url + " \r\n" + "工具描述: " + "" + description
+                    if load_config()[0] == "dingding":
+                        dingding(text, body,load_config()[2],load_config()[3])
+                    if load_config()[0] == "server":
+                        server(text, body,load_config()[2])
+                    if load_config()[0] == "pushplus":
+                        pushplus(text, body,load_config()[2])
+                    if load_config()[0] == "tgbot":
+                        tgbot(text,body,load_config()[2],load_config()[3])
+    except Exception as e:
+        print(e, "github链接不通")
+
+#获取用户或者组织信息插入到数据库
+def user_insert_into_sqlite3(repo_name):
+    conn = sqlite3.connect('data.db')
+    cur = conn.cursor()
+    sql_grammar = "SELECT repo_name FROM user_monitor WHERE repo_name = '{}';".format(repo_name)
+    Verify = len(list(cur.execute(sql_grammar)))
+    if Verify == 0:
+        print("[+] 用户仓库表数据库里不存在{}".format(repo_name))
+        cur.execute("INSERT INTO user_monitor (repo_name) VALUES ('{}')".format(repo_name))
+        print("user_insert_into_sqlite3 函数: {}插入数据成功！".format(repo_name))
+    else:
+        print("[-] 用户仓库表数据库里存在{}".format(repo_name))
+    conn.commit()
+    conn.close()
+    return Verify
+
 #获取更新信息并发送到对应社交软件
 def send_body(url,query_pushed_at,query_tag_name):
     # 考虑到有的工具没有 releases, 则通过 commits 记录获取更新描述
@@ -533,7 +586,7 @@ def get_cve_des_zh(cve):
         response = requests.get(query_cve_url, headers=github_headers, timeout=10)
         html = etree.HTML(response.text)
         des = html.xpath('//*[@id="GeneratedTable"]/table//tr[4]/td/text()')[0].strip()
-        if load_config()[4]:
+        if load_config()[-1]:
             return translate(des)
         return des
     except Exception as e:
@@ -599,14 +652,18 @@ def sendKeywordNews(keyword, data):
 
 #main函数
 if __name__ == '__main__':
-    print("cve 和 github 发布工具 监控中 ...")
+    print("cve 、github 工具 和 大佬仓库 监控中 ...")
     #初始化部分
     create_database()
 
     while True:
-        tools_list, keyword_list = load_tools_list()
+        tools_list, keyword_list, user_list = load_tools_list()
         tools_data = get_pushed_at_time(tools_list)
         tools_insert_into_sqlite3(tools_data)   # 获取文件中的工具列表，并从 github 获取相关信息，存储下来
+
+        print("\r\n\t\t  用户仓库监控 \t\t\r\n")
+        for user in user_list:
+            getUserRepos(user)
         #CVE部分
         print("\r\n\t\t  CVE 监控 \t\t\r\n")
         cve_data = getNews()
@@ -618,6 +675,7 @@ if __name__ == '__main__':
         print("\r\n\t\t  关键字监控 \t\t\r\n")
         # 关键字监控 , 最好不要太多关键字，防止 github 次要速率限制  https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits=
         for keyword in keyword_list:
+             time.sleep(1)  # 每个关键字停 1s ，防止关键字过多导致速率限制
              keyword_data = getKeywordNews(keyword)
 
              if len(keyword_data) > 0:
@@ -627,9 +685,8 @@ if __name__ == '__main__':
                     keyword_insert_into_sqlite3(today_keyword_data)
 
         print("\r\n\t\t  红队工具监控 \t\t\r\n")
-        # 红队工具部分
+        time.sleep(5*60)
-        time.sleep(3*60)
+        tools_list_new, keyword_list, user_list = load_tools_list()
-        tools_list_new, keyword_list = load_tools_list()
         data2 = get_pushed_at_time(tools_list_new)      # 再次从文件中获取工具列表，并从 github 获取相关信息,
         data3 = get_tools_update_list(data2)        # 与 3 分钟前数据进行对比，如果在三分钟内有新增工具清单或者工具有更新则通知一下用户
         for i in range(len(data3)):

tools_list.yaml

@@ -3,7 +3,6 @@ tools_list:
   - https://api.github.com/repos/rebeyond/Behinder
   - https://api.github.com/repos/AntSwordProject/antSword
   - https://api.github.com/repos/j1anFen/shiro_attack
-  - https://api.github.com/repos/yhy0/ExpDemo-JavaFX
   - https://api.github.com/repos/yhy0/github-cve-monitor
   - https://api.github.com/repos/gentilkiwi/mimikatz
   - https://api.github.com/repos/ehang-io/nps
@@ -16,8 +15,15 @@ tools_list:
 
 keyword_list:
  - Sql注入
- - rce
  - cnvd
  - 未授权
- - 注入
+
- - 命令执行
+user_list:
+ - yhy0
+ - su18
+ - BeichenDream
+ - phith0n
+ - zhzyker
+ - lijiejie
+ - projectdiscovery
+ - HavocFramework