[fix] 修复关键字监控入库错误，新增关键字入库前检查是否有 cve 防止与 cve 监控重复推送

config.yaml

@@ -1,12 +1,12 @@
 all_config:
  github_token: xxxxxx
  dingding:
-   - enable: 0
+   - enable: 1
    - webhook: xxxxxxx
    - secretKey: xxxxxxx
    - app_name: dingding
  feishu:
-   - enable: 1
+   - enable: 0
    - webhook: xxx
    - app_name: feishu
  server:
@@ -25,4 +25,4 @@ all_config:
  translate:
   - enable: 0
  black_user:
-   - AlAIAL90
+   - AlAIAL90

github_cve_monitor.py

@@ -143,11 +143,8 @@ def getKeywordNews(keyword):
     today_keyword_info_tmp = []
     try:
         # 抓取本年的
-        year = datetime.datetime.now().year
         api = "https://api.github.com/search/repositories?q={}&sort=updated".format(keyword)
         json_str = requests.get(api, headers=github_headers, timeout=10).json()
-        # cve_total_count = json_str['total_count']
-        # cve_description = json_str['items'][0]['description']
         today_date = datetime.date.today()
         n = len(json_str['items'])
         if n > 20:
@@ -161,8 +158,9 @@ def getKeywordNews(keyword):
                     pushed_at = re.findall('\d{4}-\d{2}-\d{2}', pushed_at_tmp)[0]
                     if pushed_at == str(today_date):
                         today_keyword_info_tmp.append({"keyword_name": keyword_name, "keyword_url": keyword_url, "pushed_at": pushed_at})
+                        print("[+] keyword: {} ,{}".format(keyword, keyword_name))
                     else:
-                        print("[-] 该{}的更新时间为{}, 不属于今天".format(keyword_name, pushed_at))
+                        print("[-] keyword: {} ,该{}的更新时间为{}, 不属于今天".format(keyword, keyword_name, pushed_at))
                 except Exception as e:
                     pass
             else:
@@ -176,7 +174,7 @@ def getKeywordNews(keyword):
 
     except Exception as e:
         print(e, "github链接不通")
-        return '', '', ''
+    return today_keyword_info_tmp
 
 #获取到的关键字仓库信息插入到数据库
 def keyword_insert_into_sqlite3(data):
@@ -208,13 +206,16 @@ def get_today_keyword_info(today_keyword_info_data):
     for i in range(len(today_keyword_info_data)):
         try:
             today_keyword_name = today_keyword_info_data[i]['keyword_name']
-            if exist_cve(today_keyword_name) == 1:
+            today_cve_name = re.findall('(CVE\-\d+\-\d+)', today_keyword_info_data[i]['keyword_name'].upper())
-                Verify = query_keyword_info_database(today_keyword_name)
+            # 如果仓库名字带有 cve-xxx-xxx, 先查询看看 cve 监控中是否存在, 防止重复推送
-                if Verify == 0:
+            if len(today_cve_name) > 0 and query_cve_info_database(today_cve_name.upper()) == 1: 
-                    print("[+] 数据库里不存在{}".format(today_keyword_name))
+                pass
-                    today_all_keyword_info.append(today_keyword_info_data[i])
+            Verify = query_keyword_info_database(today_keyword_name)
-                else:
+            if Verify == 0:
-                    print("[-] 数据库里存在{}".format(today_keyword_name))
+                print("[+] 数据库里不存在{}".format(today_keyword_name))
+                today_all_keyword_info.append(today_keyword_info_data[i])
+            else:
+                print("[-] 数据库里存在{}".format(today_keyword_name))
         except Exception as e:
             pass
     return today_all_keyword_info
@@ -618,6 +619,7 @@ if __name__ == '__main__':
         # 关键字监控 , 最好不要太多关键字，防止 github 次要速率限制  https://docs.github.com/en/rest/overview/resources-in-the-rest-api#secondary-rate-limits=
         for keyword in keyword_list:
              keyword_data = getKeywordNews(keyword)
+
              if len(keyword_data) > 0:
                 today_keyword_data = get_today_keyword_info(keyword_data)
                 if len(today_keyword_data) > 0:

tools_list.yaml

@@ -15,6 +15,7 @@ tools_list:
   - https://api.github.com/repos/EdgeSecurityTeam/Vulnerability
 
 keyword_list:
+ - Sql注入
  - rce
  - cnvd
  - 未授权