添加了对github-release情况的监测
This commit is contained in:
parent
91d273810d
commit
e594ca1711
6
Core.py
6
Core.py
@ -134,12 +134,14 @@ def send_job_SX():
|
||||
|
||||
def send_job_github(time_1):
|
||||
github_main()
|
||||
result_github_1_long, result_github_2_long, result_github_3_long = Src_github(time_1, False)
|
||||
result_github_1_short, result_github_2_short, result_github_3_short = Src_github(time_1, True)
|
||||
result_github_1_long, result_github_2_long, result_github_3_long, result_github_4_long = Src_github(time_1, False)
|
||||
result_github_1_short, result_github_2_short, result_github_3_short, result_github_4_short = Src_github(time_1, True)
|
||||
webhook_url, timestamp, sign = gen_sign()
|
||||
check_avaliable(result_github_1_long, result_github_1_short, "Github项目监控-关键词监控", webhook_url, timestamp, sign)
|
||||
check_avaliable(result_github_2_long, result_github_2_short, "Github项目监控-项目更新情况", webhook_url, timestamp, sign)
|
||||
webhook_url, timestamp, sign = gen_sign()
|
||||
check_avaliable(result_github_3_long, result_github_3_short, "Github项目监控-大佬工具", webhook_url, timestamp, sign)
|
||||
check_avaliable(result_github_4_long, result_github_4_short, "Github项目监控-项目版本发布监测", webhook_url, timestamp, sign)
|
||||
|
||||
# 探测rss源状态
|
||||
def check_rss_status(url):
|
||||
|
||||
@ -31,6 +31,13 @@ def create_database():
|
||||
keyword TEXT,
|
||||
link2 TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS releases (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
link TEXT,
|
||||
pubDate DATETIME,
|
||||
author TEXT,
|
||||
keyword TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
title TEXT,
|
||||
@ -66,6 +73,13 @@ def insert_data():
|
||||
if not isinstance(data_repo, list):
|
||||
raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！")
|
||||
|
||||
if not os.path.exists('./JSON/github_release.json'):
|
||||
raise FileNotFoundError(f"github_release文件不存在，请检查程序是否运行正常！")
|
||||
with open('./JSON/github_release.json', 'r', encoding='utf-8') as file:
|
||||
data_release = json.load(file)
|
||||
if not isinstance(data_release, list):
|
||||
raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！")
|
||||
|
||||
if not os.path.exists('./JSON/github_user.json'):
|
||||
raise FileNotFoundError(f"github_user文件不存在，请检查程序是否运行正常！")
|
||||
with open('./JSON/github_user.json', 'r', encoding='utf-8') as file:
|
||||
@ -134,6 +148,28 @@ def insert_data():
|
||||
INSERT INTO repos (title, link, description, pubDate, author, link2, keyword)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
''', (entry['title'], entry['link'], entry['description'], formatted_pub_date, entry['author'], entry['link2'], entry['keyword']))
|
||||
|
||||
for index, item in enumerate(data_release, start=1):
|
||||
entry = {
|
||||
"id": index,
|
||||
"link": item.get("link", ""),
|
||||
"pubDate": item.get("published_at", ""),
|
||||
"author": item.get("author", ""),
|
||||
"keyword": item.get("keyword", "")
|
||||
}
|
||||
try:
|
||||
# 解析 pubDate 字符串为 datetime 对象
|
||||
pub_date = datetime.strptime(entry['pubDate'], '%Y-%m-%dT%H:%M:%SZ')
|
||||
# 格式化 pubDate 为所需的格式
|
||||
formatted_pub_date = pub_date.strftime('%Y-%m-%d %H:%M:%S')
|
||||
except ValueError:
|
||||
# 如果解析失败，使用原始 pubDate 字符串
|
||||
formatted_pub_date = entry['pubDate']
|
||||
|
||||
cursor.execute('''
|
||||
INSERT INTO releases (link, pubDate, author, keyword)
|
||||
VALUES (?, ?, ?, ?)
|
||||
''', (entry['link'], formatted_pub_date, entry['author'], entry['keyword']))
|
||||
|
||||
|
||||
# 插入 users 数据
|
||||
@ -215,14 +251,6 @@ def select_articles(e_hour):
|
||||
ORDER BY pubDate DESC
|
||||
LIMIT 5
|
||||
''', (two_months_ago.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S')))
|
||||
|
||||
# 查询最近的3条未被标记为True的消息
|
||||
# cursor.execute('''
|
||||
# SELECT * FROM users
|
||||
# WHERE is_sended IS NULL
|
||||
# ORDER BY pubDate DESC
|
||||
# LIMIT 5
|
||||
# ''')
|
||||
|
||||
result_3 = cursor.fetchall()
|
||||
# print(results)
|
||||
@ -238,15 +266,23 @@ def select_articles(e_hour):
|
||||
|
||||
conn.commit() # 提交事务
|
||||
|
||||
cursor.execute('''
|
||||
SELECT * FROM releases
|
||||
WHERE pubDate BETWEEN ? AND ?
|
||||
''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S')))
|
||||
|
||||
result_4 = cursor.fetchall()
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
return result_1, result_2, result_3
|
||||
return result_1, result_2, result_3, result_4
|
||||
|
||||
def clear_table():
|
||||
conn = sqlite3.connect('./db/github.db')
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DELETE FROM repos')
|
||||
cursor.execute('DELETE FROM releases')
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
@ -287,6 +323,10 @@ def get_filtered_articles(entries, Is_short, choice):
|
||||
result += f"项目描述：{entry[3]}\n"
|
||||
result += f"上传时间：{entry[4]}\n"
|
||||
result += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章
|
||||
elif choice == 4:
|
||||
result += f"【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
|
||||
result += f"发布时间：{entry[2]}\n"
|
||||
result += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章
|
||||
elif Is_short == True:
|
||||
if choice == 1:
|
||||
result += f"关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n"
|
||||
@ -304,6 +344,10 @@ def get_filtered_articles(entries, Is_short, choice):
|
||||
result += f"项目描述：{entry[3]}\n"
|
||||
result += f"上传时间：{entry[4]}\n"
|
||||
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
|
||||
elif choice == 4:
|
||||
result += f"【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
|
||||
result += f"发布时间：{entry[2]}\n"
|
||||
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
|
||||
|
||||
if choice == 1:
|
||||
record += f"#### 关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n"
|
||||
@ -322,6 +366,10 @@ def get_filtered_articles(entries, Is_short, choice):
|
||||
record += f"**项目描述**：{entry[3]}\n"
|
||||
record += f"**上传时间**：{entry[4]}\n"
|
||||
record += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章
|
||||
elif choice == 4:
|
||||
record += f"#### 【{entry[3]}】为[{entry[4]}]({entry[1]})发布了新版本，请及时查收！\n"
|
||||
record += f"**发布时间**：{entry[2]}\n"
|
||||
record += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章
|
||||
|
||||
record_md(record)
|
||||
return result
|
||||
@ -339,7 +387,7 @@ def Src_github(e_hour, Is_short):
|
||||
insert_data()
|
||||
|
||||
# 查询指定时间段内的数据
|
||||
keyword_data, repo_data, user_data = select_articles(e_hour)
|
||||
keyword_data, repo_data, user_data, release_data = select_articles(e_hour)
|
||||
|
||||
if keyword_data:
|
||||
result_1 = get_filtered_articles(keyword_data, Is_short, 1)
|
||||
@ -353,17 +401,23 @@ def Src_github(e_hour, Is_short):
|
||||
result_3 = get_filtered_articles(user_data, Is_short, 3)
|
||||
else:
|
||||
result_3 = ""
|
||||
return result_1, result_2, result_3
|
||||
if release_data:
|
||||
result_4 = get_filtered_articles(release_data, Is_short, 4)
|
||||
else:
|
||||
result_4 = ""
|
||||
return result_1, result_2, result_3, result_4
|
||||
|
||||
if __name__ == "__main__":
|
||||
result_1, result_2, result_3 = Src_github(24, False)
|
||||
result_1, result_2, result_3, result_4 = Src_github(24000, False)
|
||||
if result_1 != "":
|
||||
print(result_1)
|
||||
elif result_2 != "":
|
||||
if result_2 != "":
|
||||
print(result_2)
|
||||
if result_3 != "":
|
||||
print(result_3)
|
||||
else:
|
||||
if result_4 != "":
|
||||
print(result_4)
|
||||
if result_1 == "" and result_2 == "" and result_3 == "" and result_4 == "":
|
||||
# 如果为空，则跳过执行
|
||||
print("-" * 10)
|
||||
print("github数据为空，跳过执行。")
|
||||
@ -17,8 +17,8 @@ tool_list: # 监控已创建的仓库是否更新
|
||||
- shadow1ng/fscan
|
||||
- SafeGroceryStore/MDUT
|
||||
- EdgeSecurityTeam/Vulnerability
|
||||
- wy876/POC
|
||||
- Vme18000yuan/FreePOC
|
||||
- wy876/POC
|
||||
|
||||
keyword_list: # 监控关键词
|
||||
- sql注入
|
||||
|
||||
@ -82,7 +82,7 @@ def github_main_keyword(key):
|
||||
}
|
||||
# print(project_info)
|
||||
all_results.append(project_info)
|
||||
time.sleep(10)
|
||||
time.sleep(5)
|
||||
|
||||
# 将所有结果写入JSON文件
|
||||
save_to_json(all_results, './JSON/github_keyword.json')
|
||||
@ -125,11 +125,43 @@ def github_main_repo(key):
|
||||
}
|
||||
# print(project_info)
|
||||
all_results.append(project_info)
|
||||
time.sleep(10)
|
||||
time.sleep(5)
|
||||
|
||||
# 将所有结果写入JSON文件
|
||||
save_to_json(all_results, './JSON/github_repo.json')
|
||||
|
||||
def github_main_release(key):
|
||||
all_results = [] # 用于存储所有结果
|
||||
for keyword in key:
|
||||
logger.info(f"github_repo:项目【{keyword}】发版情况获取开始。")
|
||||
api_node = "https://api.github.com/repos/{}/releases?per_page=1".format(keyword)
|
||||
result = fetch_rss(api_node)
|
||||
if not result:
|
||||
logger.warning(f"github_repo:项目【{keyword}】不存在版本发布情况。")
|
||||
time.sleep(5)
|
||||
continue
|
||||
|
||||
# print(result)
|
||||
# print(keyword)
|
||||
commit = result[0] # 获取最新的提交记录
|
||||
author = commit['author']['login']
|
||||
published_at = commit['published_at']
|
||||
link = commit['html_url']
|
||||
|
||||
# 将每个项目的详细信息存储在一个字典中
|
||||
project_info = {
|
||||
'link': link,
|
||||
'published_at': published_at,
|
||||
'author': author,
|
||||
'keyword': keyword
|
||||
}
|
||||
# print(project_info)
|
||||
all_results.append(project_info)
|
||||
time.sleep(5)
|
||||
|
||||
# 将所有结果写入JSON文件
|
||||
save_to_json(all_results, './JSON/github_release.json')
|
||||
|
||||
def github_main_user(key):
|
||||
all_results = [] # 用于存储所有结果
|
||||
for keyword in key:
|
||||
@ -172,7 +204,7 @@ def github_main_user(key):
|
||||
}
|
||||
# print(project_info)
|
||||
all_results.append(project_info)
|
||||
time.sleep(10)
|
||||
time.sleep(5)
|
||||
|
||||
# 将所有结果写入JSON文件
|
||||
save_to_json(all_results, './JSON/github_user.json')
|
||||
@ -182,6 +214,7 @@ def github_main():
|
||||
github_main_keyword(keyword_list)
|
||||
if tool_list:
|
||||
github_main_repo(tool_list)
|
||||
github_main_release(tool_list)
|
||||
if user_list:
|
||||
github_main_user(user_list)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user