添加了对github-release情况的监测

Core.py

@@ -134,12 +134,14 @@ def send_job_SX():
 
 def send_job_github(time_1):
     github_main()
-    result_github_1_long, result_github_2_long, result_github_3_long = Src_github(time_1, False)
+    result_github_1_long, result_github_2_long, result_github_3_long, result_github_4_long = Src_github(time_1, False)
-    result_github_1_short, result_github_2_short, result_github_3_short = Src_github(time_1, True)
+    result_github_1_short, result_github_2_short, result_github_3_short, result_github_4_short = Src_github(time_1, True)
     webhook_url, timestamp, sign = gen_sign()
     check_avaliable(result_github_1_long, result_github_1_short, "Github项目监控-关键词监控", webhook_url, timestamp, sign)
     check_avaliable(result_github_2_long, result_github_2_short, "Github项目监控-项目更新情况", webhook_url, timestamp, sign)
+    webhook_url, timestamp, sign = gen_sign()
     check_avaliable(result_github_3_long, result_github_3_short, "Github项目监控-大佬工具", webhook_url, timestamp, sign)
+    check_avaliable(result_github_4_long, result_github_4_short, "Github项目监控-项目版本发布监测", webhook_url, timestamp, sign)
 
 # 探测rss源状态
 def check_rss_status(url):

GotoSend/github.py

@@ -31,6 +31,13 @@ def create_database():
         keyword TEXT,
         link2 TEXT
     );
+    CREATE TABLE IF NOT EXISTS releases (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        link TEXT,
+        pubDate DATETIME,
+        author TEXT,
+        keyword TEXT
+    );
     CREATE TABLE IF NOT EXISTS users (
         id INTEGER PRIMARY KEY AUTOINCREMENT,
         title TEXT,
@@ -66,6 +73,13 @@ def insert_data():
     if not isinstance(data_repo, list):
         raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！")
     
+    if not os.path.exists('./JSON/github_release.json'):
+        raise FileNotFoundError(f"github_release文件不存在，请检查程序是否运行正常！")    
+    with open('./JSON/github_release.json', 'r', encoding='utf-8') as file:
+        data_release = json.load(file)
+    if not isinstance(data_release, list):
+        raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！")
+    
     if not os.path.exists('./JSON/github_user.json'):
         raise FileNotFoundError(f"github_user文件不存在，请检查程序是否运行正常！")
     with open('./JSON/github_user.json', 'r', encoding='utf-8') as file:
@@ -134,6 +148,28 @@ def insert_data():
             INSERT INTO repos (title, link, description, pubDate, author, link2, keyword)
             VALUES (?, ?, ?, ?, ?, ?, ?)
         ''', (entry['title'], entry['link'], entry['description'], formatted_pub_date, entry['author'], entry['link2'], entry['keyword']))
+
+    for index, item in enumerate(data_release, start=1):
+        entry = {
+            "id": index,
+            "link": item.get("link", ""),
+            "pubDate": item.get("published_at", ""),
+            "author": item.get("author", ""),
+            "keyword": item.get("keyword", "")
+        }
+        try:
+            # 解析 pubDate 字符串为 datetime 对象
+            pub_date = datetime.strptime(entry['pubDate'], '%Y-%m-%dT%H:%M:%SZ')
+            # 格式化 pubDate 为所需的格式
+            formatted_pub_date = pub_date.strftime('%Y-%m-%d %H:%M:%S')
+        except ValueError:
+            # 如果解析失败，使用原始 pubDate 字符串
+            formatted_pub_date = entry['pubDate']
+        
+        cursor.execute('''
+            INSERT INTO releases (link, pubDate, author, keyword)
+            VALUES (?, ?, ?, ?)
+        ''', (entry['link'], formatted_pub_date, entry['author'], entry['keyword']))
     
 
     # 插入 users 数据
@@ -215,14 +251,6 @@ def select_articles(e_hour):
         ORDER BY pubDate DESC
         LIMIT 5
     ''', (two_months_ago.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S')))
-
-    # 查询最近的3条未被标记为True的消息
-    # cursor.execute('''
-    #     SELECT * FROM users
-    #     WHERE is_sended IS NULL
-    #     ORDER BY pubDate DESC
-    #     LIMIT 5
-    # ''')
     
     result_3 = cursor.fetchall()
     # print(results)
@@ -238,15 +266,23 @@ def select_articles(e_hour):
         
         conn.commit()  # 提交事务
 
+    cursor.execute('''
+            SELECT * FROM releases
+            WHERE pubDate BETWEEN ? AND ?
+        ''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S')))
+
+    result_4 = cursor.fetchall()
+
     cursor.close()
     conn.close()
     
-    return result_1, result_2, result_3
+    return result_1, result_2, result_3, result_4
 
 def clear_table():
     conn = sqlite3.connect('./db/github.db')
     cursor = conn.cursor()
     cursor.execute('DELETE FROM repos')
+    cursor.execute('DELETE FROM releases')
     conn.commit()
     conn.close()
 
@@ -287,6 +323,10 @@ def get_filtered_articles(entries, Is_short, choice):
                 result += f"项目描述：{entry[3]}\n"
                 result += f"上传时间：{entry[4]}\n"
                 result += "\n" + "-" * 10 + "\n"  # 添加分隔线以便区分不同文章
+            elif choice == 4:
+                result += f"【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
+                result += f"发布时间：{entry[2]}\n"
+                result += "\n" + "-" * 10 + "\n"  # 添加分隔线以便区分不同文章
         elif Is_short == True:
             if choice == 1:
                 result += f"关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n"
@@ -304,6 +344,10 @@ def get_filtered_articles(entries, Is_short, choice):
                 result += f"项目描述：{entry[3]}\n"
                 result += f"上传时间：{entry[4]}\n"
                 result += "\n" + "-" * 3 + "\n"  # 添加分隔线以便区分不同文章
+            elif choice == 4:
+                result += f"【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
+                result += f"发布时间：{entry[2]}\n"
+                result += "\n" + "-" * 3 + "\n"  # 添加分隔线以便区分不同文章
 
         if choice == 1:
             record += f"#### 关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n"
@@ -322,6 +366,10 @@ def get_filtered_articles(entries, Is_short, choice):
             record += f"**项目描述**：{entry[3]}\n"
             record += f"**上传时间**：{entry[4]}\n"
             record += "\n" + "-" * 10 + "\n"  # 添加分隔线以便区分不同文章
+        elif choice == 4:
+            record += f"#### 【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
+            record += f"**发布时间**：{entry[2]}\n"
+            record += "\n" + "-" * 10 + "\n"  # 添加分隔线以便区分不同文章
 
     record_md(record)
     return result
@@ -339,7 +387,7 @@ def Src_github(e_hour, Is_short):
     insert_data()
 
     # 查询指定时间段内的数据
-    keyword_data, repo_data, user_data = select_articles(e_hour)
+    keyword_data, repo_data, user_data, release_data = select_articles(e_hour)
 
     if keyword_data:
         result_1 = get_filtered_articles(keyword_data, Is_short, 1)
@@ -353,17 +401,23 @@ def Src_github(e_hour, Is_short):
         result_3 = get_filtered_articles(user_data, Is_short, 3)
     else:
         result_3 = ""
-    return result_1, result_2, result_3
+    if release_data:
+        result_4 = get_filtered_articles(release_data, Is_short, 4)
+    else:
+        result_4 = ""
+    return result_1, result_2, result_3, result_4
 
 if __name__ == "__main__":
-    result_1, result_2, result_3 = Src_github(24, False)
+    result_1, result_2, result_3, result_4 = Src_github(24000, False)
     if result_1 != "":
         print(result_1)
-    elif result_2 != "":
+    if result_2 != "":
         print(result_2)
     if result_3 != "":
         print(result_3)
-    else:
+    if result_4 != "":
+        print(result_4)
+    if result_1 == "" and result_2 == "" and result_3 == "" and result_4 == "":
         # 如果为空，则跳过执行
         print("-" * 10)
         print("github数据为空，跳过执行。")

config/github_config.yaml

@@ -17,8 +17,8 @@ tool_list:  # 监控已创建的仓库是否更新
   - shadow1ng/fscan
   - SafeGroceryStore/MDUT
   - EdgeSecurityTeam/Vulnerability
-  - wy876/POC
   - Vme18000yuan/FreePOC
+  - wy876/POC
 
 keyword_list: # 监控关键词
   - sql注入

spider/github.py

@@ -82,7 +82,7 @@ def github_main_keyword(key):
             }
             # print(project_info)
             all_results.append(project_info)
-        time.sleep(10)
+        time.sleep(5)
     
     # 将所有结果写入JSON文件
     save_to_json(all_results, './JSON/github_keyword.json')
@@ -125,11 +125,43 @@ def github_main_repo(key):
         }
         # print(project_info)
         all_results.append(project_info)
-        time.sleep(10)
+        time.sleep(5)
 
     # 将所有结果写入JSON文件
     save_to_json(all_results, './JSON/github_repo.json')
 
+def github_main_release(key):
+    all_results = []  # 用于存储所有结果
+    for keyword in key:
+        logger.info(f"github_repo:项目【{keyword}】发版情况获取开始。")
+        api_node = "https://api.github.com/repos/{}/releases?per_page=1".format(keyword)
+        result = fetch_rss(api_node)
+        if not result:
+            logger.warning(f"github_repo:项目【{keyword}】不存在版本发布情况。")
+            time.sleep(5)
+            continue
+        
+        # print(result)
+        # print(keyword)
+        commit = result[0]  # 获取最新的提交记录
+        author = commit['author']['login']
+        published_at = commit['published_at']
+        link = commit['html_url']
+
+        # 将每个项目的详细信息存储在一个字典中
+        project_info = {
+            'link': link,
+            'published_at': published_at,
+            'author': author,
+            'keyword': keyword
+        }
+        # print(project_info)
+        all_results.append(project_info)
+        time.sleep(5)
+
+    # 将所有结果写入JSON文件
+    save_to_json(all_results, './JSON/github_release.json')
+
 def github_main_user(key):
     all_results = []  # 用于存储所有结果
     for keyword in key:
@@ -172,7 +204,7 @@ def github_main_user(key):
             }
             # print(project_info)
             all_results.append(project_info)
-        time.sleep(10)
+        time.sleep(5)
     
     # 将所有结果写入JSON文件
     save_to_json(all_results, './JSON/github_user.json')
@@ -182,6 +214,7 @@ def github_main():
         github_main_keyword(keyword_list)
     if tool_list:
         github_main_repo(tool_list)
+        github_main_release(tool_list)
     if user_list:
         github_main_user(user_list)