From 353f1db25d8653487d0f737436d2660a4f89254e Mon Sep 17 00:00:00 2001
From: MasonLiu <2857911564@qq.com>
Date: Wed, 25 Dec 2024 18:40:20 +0800
Subject: [PATCH] update
---
Core.py | 98 ++++++++----------
GotoSend/M_4hou.py | 9 +-
GotoSend/anquanke.py | 7 +-
GotoSend/doonsec.py | 12 +--
GotoSend/freebuf.py | 11 +-
GotoSend/qianxin.py | 11 +-
GotoSend/seebug.py | 11 +-
GotoSend/xianzhi.py | 11 +-
README.md | 7 +-
SendCore/FeishuSendBot.py | 2 +-
SendCore/MailSendBot.py | 2 +-
SendCore/QiweiSendBot.py | 2 +-
UpdateLOG.md | 19 ++--
.../__pycache__/check_config.cpython-312.pyc | Bin 0 -> 3029 bytes
config/check_config.py | 63 +++++++++++
config.yaml => config/config.yaml | 16 ++-
log/{spider.log => app.log} | 0
web/app.py | 38 +++++--
web/templates/index.html | 4 +-
19 files changed, 205 insertions(+), 118 deletions(-)
create mode 100644 config/__pycache__/check_config.cpython-312.pyc
create mode 100644 config/check_config.py
rename config.yaml => config/config.yaml (52%)
rename log/{spider.log => app.log} (100%)
diff --git a/Core.py b/Core.py
index 3a10398..2975592 100644
--- a/Core.py
+++ b/Core.py
@@ -24,6 +24,7 @@ from GotoSend.xianzhi import Src_xianzhi
from GotoSend.freebuf import Src_freebuf
from GotoSend.qianxin import Src_qianxin
from GotoSend.seebug import Src_seebug
+from config.check_config import get_core_config, get_debug_config
from loguru import logger
# 清除所有已有的日志记录器配置
@@ -35,21 +36,20 @@ logger.add("./log/core.log",
compression="zip",
encoding="utf-8")
# shell终端打印日志
-# logger.add(lambda msg: print(msg),
-# format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name}:{function}:{line} - {message}")
+debug = get_debug_config()
+if debug == "True":
+ logger.add(lambda msg: print(msg),
+ format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name}:{function}:{line} - {message}")
-# 加载参数
-with open('./config.yaml', 'r', encoding="utf-8") as file:
- config = yaml.safe_load(file)
-# sleep_time = int(f"{config['sleep_time']}")
-e_hour = int(f"{config['e_hour']}")
-choice = int(f"{config['circle']}")
-fs_activate = f"{config['fs_activate']}"
-wx_activate = f"{config['wx_activate']}"
-ding_activate = f"{config['ding_activate']}"
-lx_activate = f"{config['lx_activate']}"
+def signal_handler(sig, frame):
+ logger.info("接收到退出信号,程序即将退出...")
+ sys.exit(0)
+# 全局变量
+signal.signal(signal.SIGINT, signal_handler) # Ctrl+C
+signal.signal(signal.SIGTERM, signal_handler) # kill命令
webhook_url_once, timestamp_once, sign_once = gen_sign()
+e_hour, choice, fs_activate, wx_activate, ding_activate, lx_activate, url_web = get_core_config()
def check_avaliable(info_long, info_short, title, webhook_url, timestamp, sign):
if info_long: # 发送完整文章相关内容
@@ -121,52 +121,17 @@ def send_job(time_1):
check_avaliable(result_seebug_long, result_seebug_short, "Seebug社区资讯", webhook_url, timestamp, sign)
if fs_activate == "True":
- send_result = SendToFeishu("[点此访问](https://info.masonliu.com)网站以查看全部文章。", "单次运行结束", webhook_url, timestamp, sign)
+ send_result = SendToFeishu(f"[点此访问]({url_web})网站以查看全部文章。", "单次运行结束", webhook_url, timestamp, sign)
logger.info(send_result)
else:
pass
if wx_activate == "True":
- send_result = SendToWX("[点此访问](https://info.masonliu.com)网站以查看全部文章。", "单次运行结束")
+ send_result = SendToWX(f"[点此访问]({url_web})网站以查看全部文章。", "单次运行结束")
logger.info(send_result)
else:
pass
logger.info("执行完毕,等待下一次执行...")
-def signal_handler(sig, frame):
- logger.info("接收到退出信号,程序即将退出...")
- sys.exit(0)
-
-
-signal.signal(signal.SIGINT, signal_handler) # Ctrl+C
-signal.signal(signal.SIGTERM, signal_handler) # kill命令
-
-
-def main_loop(choice):
- if choice == 1:
- while True:
- try:
- # 执行任务
- send_job(e_hour)
- time.sleep(e_hour * 60 * 60 - 3 * 60)
-
- except Exception as e:
- logger.error(f"发生错误: {e}, 程序已暂停")
- # result = SendToFeishu(f"发生错误: {e}, 程序已退出", "报错信息")
- # logger.info(result)
- exit()
-
- elif choice == 0:
- # 设置每天的特定时间点执行job函数
- schedule.every().day.at("09:00").do(send_job, 12)
- schedule.every().day.at("12:00").do(send_job, 3)
- schedule.every().day.at("15:00").do(send_job, 3)
- schedule.every().day.at("18:00").do(send_job, 3)
- schedule.every().day.at("21:00").do(send_job, 3)
-
- while True:
- schedule.run_pending()
- time.sleep(60) # 每分钟检查一次是否有任务需要执行
-
# 探测rss源状态
def check_rss_status(url):
try:
@@ -202,9 +167,33 @@ def test_rss_source():
return rss_info
-if __name__ == "__main__":
- print("程序正在运行当中。")
- time.sleep(5) # 添加短暂的延迟
+def main_loop(choice):
+ if choice == 1:
+ while True:
+ try:
+ # 执行任务
+ send_job(e_hour)
+ time.sleep(e_hour * 60 * 60 - 3 * 60)
+
+ except Exception as e:
+ logger.error(f"发生错误: {e}, 程序已暂停")
+ # result = SendToFeishu(f"发生错误: {e}, 程序已退出", "报错信息")
+ # logger.info(result)
+ exit()
+
+ elif choice == 0:
+ # 设置每天的特定时间点执行job函数
+ schedule.every().day.at("09:00").do(send_job, 12)
+ schedule.every().day.at("12:00").do(send_job, 3)
+ schedule.every().day.at("15:00").do(send_job, 3)
+ schedule.every().day.at("18:00").do(send_job, 3)
+ schedule.every().day.at("21:00").do(send_job, 3)
+
+ while True:
+ schedule.run_pending()
+ time.sleep(60) # 每分钟检查一次是否有任务需要执行
+
+def send_first_message():
rss_info = test_rss_source()
start_info = ""
start_info += "程序已启动,当前时间为:" + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n"
@@ -229,9 +218,14 @@ if __name__ == "__main__":
logger.info(result)
else:
pass
+
+if __name__ == "__main__":
+ logger.info("程序正在运行当中。")
+ time.sleep(5) # 添加短暂的延迟
# 首次运行先暂停两分钟
# time.sleep(2 * 60)
# 主程序
+ send_first_message()
main_loop(choice)
\ No newline at end of file
diff --git a/GotoSend/M_4hou.py b/GotoSend/M_4hou.py
index 15d36bc..ea4c82f 100644
--- a/GotoSend/M_4hou.py
+++ b/GotoSend/M_4hou.py
@@ -112,17 +112,16 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"作者:{entry[5]}\n文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[4]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n作者:{entry[5]}\n"
+ result += f"上传时间:{entry[4]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
if Is_short == True:
- result += f"文章:{entry[1]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
result += f"链接:{entry[2]}\n上传时间:{entry[4]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**作者**:{entry[5]}\n"
- record += f"**链接**:{entry[2]}\n"
record += f"**上传时间**:{entry[4]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
record_md(record)
diff --git a/GotoSend/anquanke.py b/GotoSend/anquanke.py
index 8eb8883..045a501 100644
--- a/GotoSend/anquanke.py
+++ b/GotoSend/anquanke.py
@@ -107,18 +107,17 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"作者:{entry[6]}\n来源:{entry[3]}\n文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n作者:{entry[6]}\n来源:{entry[3]}\n"
+ result += f"上传时间:{entry[5]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
elif Is_short == True:
result += f"文章:{entry[1]}\n"
result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**作者**:{entry[6]}\n"
record += f"**来源**:{entry[3]}\n"
- record += f"**链接**:{entry[2]}\n"
record += f"**上传时间**:{entry[5]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
record_md(record)
diff --git a/GotoSend/doonsec.py b/GotoSend/doonsec.py
index 4751700..74b98df 100644
--- a/GotoSend/doonsec.py
+++ b/GotoSend/doonsec.py
@@ -114,18 +114,18 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"作者:{entry[5]}\n文章:{entry[1]}\n"
- result += f"链接:[点此访问]({entry[2]})\n上传时间:{entry[4]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"作者:{entry[5]}\n"
+ result += f"上传时间:{entry[4]}\n"
result += f"简介:{entry[3]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
if Is_short == True:
- result += f"文章:{entry[1]}\n"
- result += f"链接:[点此访问]({entry[2]})\n上传时间:{entry[4]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"上传时间:{entry[4]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**作者**:{entry[5]}\n"
- record += f"**链接**:[点此访问]({entry[2]})\n"
record += f"**上传时间**:{entry[4]}\n"
record += f"**简介**:{entry[3]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
diff --git a/GotoSend/freebuf.py b/GotoSend/freebuf.py
index d97c36c..24a0ec8 100644
--- a/GotoSend/freebuf.py
+++ b/GotoSend/freebuf.py
@@ -113,17 +113,16 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"类型:{entry[5]}\n文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[4]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n类型:{entry[5]}\n"
+ result += f"上传时间:{entry[4]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
elif Is_short == True:
- result += f"文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[4]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"上传时间:{entry[4]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**类型**:{entry[5]}\n"
- record += f"**链接**:{entry[2]}\n"
record += f"**上传时间**:{entry[4]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
record_md(record)
diff --git a/GotoSend/qianxin.py b/GotoSend/qianxin.py
index 9f89757..62bc2b9 100644
--- a/GotoSend/qianxin.py
+++ b/GotoSend/qianxin.py
@@ -104,18 +104,17 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"来源:{entry[3]}\n文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n来源:{entry[3]}\n"
+ result += f"上传时间:{entry[5]}\n"
result += f"描述:{entry[4]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
if Is_short == False:
- result += f"文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"上传时间:{entry[5]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**来源**:{entry[3]}\n"
- record += f"**链接**:{entry[2]}\n"
record += f"**上传时间**:{entry[5]}\n"
record += f"**描述**:{entry[4]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
diff --git a/GotoSend/seebug.py b/GotoSend/seebug.py
index 86f9449..67657ce 100644
--- a/GotoSend/seebug.py
+++ b/GotoSend/seebug.py
@@ -111,18 +111,17 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"类型:{entry[3]}\n文章:{entry[1]}"
- result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n类型:{entry[3]}\n"
+ result += f"上传时间:{entry[5]}\n"
result += f"{entry[4]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
if Is_short == True:
- result += f"文章:{entry[1]}"
- result += f"链接:{entry[2]}\n上传时间:{entry[5]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})"
+ result += f"上传时间:{entry[5]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**类型**:{entry[3]}\n"
- record += f"**链接**:{entry[2]}\n"
record += f"**上传时间**:{entry[5]}\n"
record += f"{entry[4]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
diff --git a/GotoSend/xianzhi.py b/GotoSend/xianzhi.py
index 807be47..7b495c5 100644
--- a/GotoSend/xianzhi.py
+++ b/GotoSend/xianzhi.py
@@ -110,16 +110,15 @@ def get_filtered_articles(entries, Is_short):
record = ""
for entry in entries:
if Is_short == False:
- result += f"文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[3]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"上传时间:{entry[3]}\n"
result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
if Is_short == False:
- result += f"文章:{entry[1]}\n"
- result += f"链接:{entry[2]}\n上传时间:{entry[3]}\n"
+ result += f"文章:[{entry[1]}]({entry[2]})\n"
+ result += f"上传时间:{entry[3]}\n"
result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章
- record += f"#### 文章:{entry[1]}\n"
- record += f"**链接**:{entry[2]}\n"
+ record += f"#### 文章:[{entry[1]}]({entry[2]})\n"
record += f"**上传时间**:{entry[3]}\n"
record += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章
record_md(record)
diff --git a/README.md b/README.md
index 18ff22c..492011e 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,11 @@ RSS订阅链接来源:https://github.com/zhengjim/Chinese-Security-RSS
使用python-json进行格式化,然后使用飞书webhook机器人进行发送
config.yaml可指定大部分可能需要的参数
+### 项目特色
+- 模块化:爬虫(获取信息部分)、分析(对获取的json信息进行筛选分析存储)、推送(推送至各渠道)、网页等各模块均可单独运行。
+- 轻量化:默认使用sqlite以及其他常见的各系统自带的库,用户仅需配置python环境,不会占用过多内存。
+- 简单化:配置好config后即可一步运行,效率极高。
+
### 日志相关
请查看./log文件夹下内容
@@ -20,7 +25,7 @@ centos: `yum install screen`
随后便可直接运行:`python Core.py`
web运行:`python ./web/app.py`
随后web网页将会在本地5000端口启动,访问即可,使用反向代理即可以域名映射到外网
-直接访问web域名即可查看历史推送,访问路径/log即可查看程序运行日志
+直接访问web域名即可查看历史推送,访问路径/log即可查看程序运行日志,/weblog查看flask日志
### 配置
首先先在飞书中创建群组,然后再创建WebHook机器人
diff --git a/SendCore/FeishuSendBot.py b/SendCore/FeishuSendBot.py
index 5f1328c..c96a396 100644
--- a/SendCore/FeishuSendBot.py
+++ b/SendCore/FeishuSendBot.py
@@ -9,7 +9,7 @@ import time
import yaml
def gen_sign():
- with open('./config.yaml', 'r', encoding="utf-8") as file:
+ with open('./config/config.yaml', 'r', encoding="utf-8") as file:
config = yaml.safe_load(file)
secret = f"{config['fs_secret']}"
# print(secret)
diff --git a/SendCore/MailSendBot.py b/SendCore/MailSendBot.py
index cc3f428..3ce06c9 100644
--- a/SendCore/MailSendBot.py
+++ b/SendCore/MailSendBot.py
@@ -7,7 +7,7 @@ from email.mime.text import MIMEText
from email.header import Header
# 加载参数
-with open('./config.yaml', 'r', encoding="utf-8") as file:
+with open('./config/config.yaml', 'r', encoding="utf-8") as file:
config = yaml.safe_load(file)
mail_host = f"{config['mail_host']}"
mail_user = f"{config['mail_user']}"
diff --git a/SendCore/QiweiSendBot.py b/SendCore/QiweiSendBot.py
index b0e7ff9..3c8342e 100644
--- a/SendCore/QiweiSendBot.py
+++ b/SendCore/QiweiSendBot.py
@@ -9,7 +9,7 @@ import hmac
import time
import yaml
-with open('./config.yaml', 'r', encoding="utf-8") as file:
+with open('./config/config.yaml', 'r', encoding="utf-8") as file:
config = yaml.safe_load(file)
webhook_url = f"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key={config['wx_key']}"
# print(webhook_url)
diff --git a/UpdateLOG.md b/UpdateLOG.md
index accb334..f570fb6 100644
--- a/UpdateLOG.md
+++ b/UpdateLOG.md
@@ -1,21 +1,26 @@
-### 从2024年12月15日开始记录
-
### 问题反馈
- 准点发送的文章在定点推送模式下可能会被遗漏推送
- 钉钉/企业微信/蓝信webhook存在字节长度限制,需要优化程序推送逻辑
-### 下一步计划
+### 下一步计划(待完成)
- 添加更多RSS订阅源(持续进行中)
+- 添加更多推送方式,如邮件、微信等
+- 添加GitHub等监测源(参考github-cve-monitor)
+- 添加Mysql作为数据库存储
+
+### 下一步计划(已完成)
- 将所有打印信息转为logging info并存档(已完成)
- 将logging info转为异步的loguru(已完成)
-- 探查异常中断原因(已发现,获取rss源时的请求未做超时)
+- 探查异常中断原因(已解决,获取rss源时的请求未做超时检测)
- 添加超时机制,防止程序异常卡死(已完成)
- 存档所有推送文章方便以后查看(已完成)
-- 添加更多推送方式,如邮件、微信等
-- 创建Web网页以展示最新推送(info.masonliu.com)
+- 创建Web网页以展示最新推送(info.masonliu.com,已完成)
### 更新日志
+#### 从2024年12月15日开始记录
- 2024年12月15日早:优化了文件结构,修复了日志记录时的小BUG,添加web展示日志功能
- 2024年12月15日晚:修复了单次运行结束时的校验错误问题
- 2024年12月18日早:添加了短文本推送机制,一定程度上解决了长字节推送问题(解决办法正在思考中)
-- 2024年12月24日晚:上传了测试0.1版本,修复了报错问题
\ No newline at end of file
+- 2024年12月24日晚:上传了测试0.1版本,修复了报错问题
+- 2024年12月25日早:优化了代码逻辑和表现
+- 2024年12月25日晚:优化了推送报文格式
\ No newline at end of file
diff --git a/config/__pycache__/check_config.cpython-312.pyc b/config/__pycache__/check_config.cpython-312.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dbefe30521f2d50f488efed4c2d701173cec671d
GIT binary patch
literal 3029
zcmcImTWk|Y6rJ%t{8&4V6DJ`Mz=%gNApz7vk&CEAAs|Q)6?_mx6&df?II+FatPLcr
zg=s+~5{Lw<#w~@WLamB|0I6zADYSg`r#}wi!)#M2Qq3b@O&cMkeD#j)O;QR3RdiN+
z=FZ&5nc2O2XMb@xYzW%@!PbyqMd)`nFq*k2j9&nu56MWz5wwT(rU;Ij_Lwj-A!2G3
z4Tq_IpePnCEK?dok~x`|1zD8Mmn{QAVfF>q)?-NKFWUx8hNUbWLwf|-4k&hDuVa)c
zwtKNIMwC#9(9V+j6(Fq$D3sKL(7Ek!JRBvyW@vRyO?4YLdbVuwMWa4dO?v7)ha=&xgyn+HM`T|9rL_wd3B$!*OzV`D))9-?m4R_
zJ^wrBbWc}RTIikI<4kQHkz74%(r
加载历史推送文件时出错!(推送历史记录为空)
'; } }) .catch(error => { @@ -119,7 +119,7 @@ const htmlContent = marked.parse(data.content); document.getElementById('markdown-content').innerHTML = htmlContent; } else { - document.getElementById('markdown-content').innerHTML = '加载历史推送文件时出错!
'; + document.getElementById('markdown-content').innerHTML = '加载历史推送文件时出错!(推送历史记录为空)
'; } }) .catch(error => {