From 99651b2ff729ec91f9fad1bb5ed7236349c742af Mon Sep 17 00:00:00 2001 From: MasonLiu <2857911564@qq.com> Date: Thu, 2 Jan 2025 17:11:11 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E7=BB=93=E6=9E=84=E8=B0=83?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Core.py | 2 +- Dev_test.py | 35 ++++----------- GotoSend/M_4hou.py | 16 +++---- GotoSend/anquanke.py | 16 +++---- GotoSend/doonsec.py | 16 +++---- GotoSend/freebuf.py | 16 +++---- GotoSend/github.py | 28 ++++++------ GotoSend/qianxin.py | 16 +++---- GotoSend/seebug.py | 16 +++---- GotoSend/sougou_wx.py | 16 +++---- GotoSend/xianzhi.py | 16 +++---- UpdateLOG.md | 3 +- {JSON => resources/JSON}/xianzhi.json | 0 {db => resources/db}/Format_to_txt.py | 0 {history => resources/history}/github.md | 0 {history => resources/history}/sec_news.md | 0 .../history}/tech_passage.md | 0 {history => resources/history}/wx_news.md | 0 {imgs => resources/imgs}/add_bot.jpg | Bin {imgs => resources/imgs}/bot_config.jpg | Bin {imgs => resources/imgs}/config.jpg | Bin {imgs => resources/imgs}/github-token.png | Bin {imgs => resources/imgs}/group.jpg | Bin {imgs => resources/imgs}/result.jpg | Bin {imgs => resources/imgs}/run.jpg | Bin {imgs => resources/imgs}/start.jpg | Bin {log => resources/log}/app.log | 0 {log => resources/log}/core.log | 0 spider/common.py | 42 +++++++++--------- spider/freebuf.py | 6 +-- spider/github.py | 8 ++-- spider/sougou_wx.py | 4 +- spider/xianzhi.py | 4 +- web/app.py | 8 ++-- 功能解释.md | 10 ++--- 35 files changed, 131 insertions(+), 147 deletions(-) rename {JSON => resources/JSON}/xianzhi.json (100%) rename {db => resources/db}/Format_to_txt.py (100%) rename {history => resources/history}/github.md (100%) rename {history => resources/history}/sec_news.md (100%) rename {history => resources/history}/tech_passage.md (100%) rename {history => resources/history}/wx_news.md (100%) rename {imgs => resources/imgs}/add_bot.jpg (100%) rename {imgs => resources/imgs}/bot_config.jpg (100%) rename {imgs => resources/imgs}/config.jpg (100%) rename {imgs => resources/imgs}/github-token.png (100%) rename {imgs => resources/imgs}/group.jpg (100%) rename {imgs => resources/imgs}/result.jpg (100%) rename {imgs => resources/imgs}/run.jpg (100%) rename {imgs => resources/imgs}/start.jpg (100%) rename {log => resources/log}/app.log (100%) rename {log => resources/log}/core.log (100%) diff --git a/Core.py b/Core.py index 6e5e7f3..02c1566 100644 --- a/Core.py +++ b/Core.py @@ -34,7 +34,7 @@ from loguru import logger # 清除所有已有的日志记录器配置 logger.remove() -logger.add("./log/core.log", +logger.add("./resources/log/core.log", format="{time:YYYY-MM-DD HH:mm:ss} - {level} - {name}:{function}:{line} - {message}", rotation="100 MB", compression="zip", diff --git a/Dev_test.py b/Dev_test.py index a68e53d..c5064e2 100644 --- a/Dev_test.py +++ b/Dev_test.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ @Author: MasonLiu -@Description: 本程序可以爬取各安全资讯源，并发送到飞书群组。 +@Description: 测试用脚本，无需关注。 """ import schedule @@ -14,11 +14,11 @@ import requests from datetime import datetime, timedelta from SendCore.FeishuSendBot import SendToFeishu, gen_sign from SendCore.QiweiSendBot import SendToWX -from media.common import run, seebug_main, M_4hou_main, anquanke_main, sec_wiki_main, huawei_main, doonsec_main, qianxin_main -from media.freebuf import freebuf_main -from media.xianzhi import xianzhi_main -from media.sougou_wx import sougou_wx_main -from media.github import github_main +from spider.common import run, seebug_main, M_4hou_main, anquanke_main, sec_wiki_main, huawei_main, doonsec_main, qianxin_main +from spider.freebuf import freebuf_main +from spider.xianzhi import xianzhi_main +from spider.sougou_wx import sougou_wx_main +from spider.github import github_main from GotoSend.M_4hou import Src_4hou from GotoSend.anquanke import Src_anquanke from GotoSend.doonsec import Src_doonsec @@ -27,28 +27,11 @@ from GotoSend.freebuf import Src_freebuf from GotoSend.qianxin import Src_qianxin from GotoSend.seebug import Src_seebug from GotoSend.sougou_wx import Src_sougou_wx -# from GotoSend.github import Src_github +from GotoSend.github import Src_github from config.check_config import get_core_config, get_debug_config, get_kewords_config from loguru import logger -# # 全局变量 -# webhook_url_once, timestamp_once, sign_once = gen_sign() -# e_hour, time_choice, choice, fs_activate, wx_activate, ding_activate, lx_activate, url_web = get_core_config() -# Sogou_WX = get_kewords_config() -# webhook_url, timestamp, sign = gen_sign() - -# sougou_wx_main(Sogou_WX) -# result_sx_long = Src_sougou_wx(False) -# result_sx_short = Src_sougou_wx(True) -# webhook_url, timestamp, sign = gen_sign() -# SendToFeishu(result_sx_long, "微信公众号关键词相关内容", webhook_url, timestamp, sign) - -github_main() -# # 测试用消息体 -# test_msg = { -# "请单件文档查看昨天讨论的方案相关飞书文档，注意作者为 **张三** 版本为 \n*002* ,版本 ~~001~~ 已经删除。文件地址是 [https://www.feishu.cn](https://www.feishu.cn),打开次数:1次" -# } - -# SendToFeishu(test_msg, "先知社区资讯递送", webhook_url, timestamp, sign) +if __name__ == "__main__": + logger.info("程序正在运行当中。") \ No newline at end of file diff --git a/GotoSend/M_4hou.py b/GotoSend/M_4hou.py index ea4c82f..e47905a 100644 --- a/GotoSend/M_4hou.py +++ b/GotoSend/M_4hou.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/4hou.db') + conn = sqlite3.connect('./resources/db/4hou.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -20,7 +20,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/4hou.db') + conn = sqlite3.connect('./resources/db/4hou.db') cursor = conn.cursor() for entry in data: try: @@ -41,11 +41,11 @@ def insert_data(data): def get_4hou_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/4hou.json'): + if not os.path.exists('./resources/JSON/4hou.json'): raise FileNotFoundError(f"4hou.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/4hou.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/4hou.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -68,7 +68,7 @@ def get_4hou_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/4hou.db') + conn = sqlite3.connect('./resources/db/4hou.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -87,13 +87,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/4hou.db') + conn = sqlite3.connect('./resources/db/4hou.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/sec_news.md"): +def record_md(result, filename="./resources/history/sec_news.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -129,7 +129,7 @@ def get_filtered_articles(entries, Is_short): def Src_4hou(e_hour, Is_short): - if not os.path.exists('./db/4hou.db'): + if not os.path.exists('./resources/db/4hou.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/anquanke.py b/GotoSend/anquanke.py index 045a501..cabf262 100644 --- a/GotoSend/anquanke.py +++ b/GotoSend/anquanke.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/anquanke.db') + conn = sqlite3.connect('./resources/db/anquanke.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -22,7 +22,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/anquanke.db') + conn = sqlite3.connect('./resources/db/anquanke.db') cursor = conn.cursor() for entry in data: cursor.execute(''' @@ -34,11 +34,11 @@ def insert_data(data): def get_anquanke_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/anquanke.json'): + if not os.path.exists('./resources/JSON/anquanke.json'): raise FileNotFoundError(f"anquanke.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/anquanke.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/anquanke.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -62,7 +62,7 @@ def get_anquanke_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/anquanke.db') + conn = sqlite3.connect('./resources/db/anquanke.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -81,13 +81,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/anquanke.db') + conn = sqlite3.connect('./resources/db/anquanke.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/sec_news.md"): +def record_md(result, filename="./resources/history/sec_news.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -125,7 +125,7 @@ def get_filtered_articles(entries, Is_short): def Src_anquanke(e_hour, Is_short): - if not os.path.exists('./db/anquanke.db'): + if not os.path.exists('./resources/db/anquanke.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/doonsec.py b/GotoSend/doonsec.py index 6e30760..f6561a8 100644 --- a/GotoSend/doonsec.py +++ b/GotoSend/doonsec.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/doonsec.db') + conn = sqlite3.connect('./resources/db/doonsec.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -21,7 +21,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/doonsec.db') + conn = sqlite3.connect('./resources/db/doonsec.db') cursor = conn.cursor() for entry in data: try: @@ -42,11 +42,11 @@ def insert_data(data): def get_doonsec_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/doonsec.json'): + if not os.path.exists('./resources/JSON/doonsec.json'): raise FileNotFoundError(f"doonsec.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/doonsec.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/doonsec.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -69,7 +69,7 @@ def get_doonsec_json(): return total_data def select_articles(e_hour, Doonsec_switch, Doonsec): - conn = sqlite3.connect('./db/doonsec.db') + conn = sqlite3.connect('./resources/db/doonsec.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -105,13 +105,13 @@ def select_articles(e_hour, Doonsec_switch, Doonsec): return results def clear_table(): - conn = sqlite3.connect('./db/doonsec.db') + conn = sqlite3.connect('./resources/db/doonsec.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/tech_passage.md"): +def record_md(result, filename="./resources/history/tech_passage.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -151,7 +151,7 @@ def get_filtered_articles(entries, Is_short): def Src_doonsec(e_hour, Is_short, Doonsec_switch, Doonsec): - if not os.path.exists('./db/doonsec.db'): + if not os.path.exists('./resources/db/doonsec.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/freebuf.py b/GotoSend/freebuf.py index 35c4c0d..e386b15 100644 --- a/GotoSend/freebuf.py +++ b/GotoSend/freebuf.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/freebuf.db') + conn = sqlite3.connect('./resources/db/freebuf.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -20,7 +20,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/freebuf.db') + conn = sqlite3.connect('./resources/db/freebuf.db') cursor = conn.cursor() for entry in data: try: @@ -41,11 +41,11 @@ def insert_data(data): def get_freebuf_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/freebuf.json'): + if not os.path.exists('./resources/JSON/freebuf.json'): raise FileNotFoundError(f"freebuf.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/freebuf.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/freebuf.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -68,7 +68,7 @@ def get_freebuf_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/freebuf.db') + conn = sqlite3.connect('./resources/db/freebuf.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -87,13 +87,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/freebuf.db') + conn = sqlite3.connect('./resources/db/freebuf.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/sec_news.md"): +def record_md(result, filename="./resources/history/sec_news.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -130,7 +130,7 @@ def get_filtered_articles(entries, Is_short): def Src_freebuf(e_hour, Is_short): - if not os.path.exists('./db/freebuf.db'): + if not os.path.exists('./resources/db/freebuf.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/github.py b/GotoSend/github.py index 1edf230..b9f069c 100644 --- a/GotoSend/github.py +++ b/GotoSend/github.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/github.db') + conn = sqlite3.connect('./resources/db/github.db') cursor = conn.cursor() cursor.executescript(''' @@ -59,35 +59,35 @@ def insert_data(): # 检查文件是否存在 # 打开并读取JSON文件 # 假设data是一个包含多个JSON对象的列表，然后校验JSON格式是否异常 - if not os.path.exists('./JSON/github_keyword.json'): + if not os.path.exists('./resources/JSON/github_keyword.json'): raise FileNotFoundError(f"github_keyword文件不存在，请检查程序是否运行正常！") - with open('./JSON/github_keyword.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/github_keyword.json', 'r', encoding='utf-8') as file: data_keyword = json.load(file) if not isinstance(data_keyword, list): raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") - if not os.path.exists('./JSON/github_repo.json'): + if not os.path.exists('./resources/JSON/github_repo.json'): raise FileNotFoundError(f"github_repo文件不存在，请检查程序是否运行正常！") - with open('./JSON/github_repo.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/github_repo.json', 'r', encoding='utf-8') as file: data_repo = json.load(file) if not isinstance(data_repo, list): raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") - if not os.path.exists('./JSON/github_release.json'): + if not os.path.exists('./resources/JSON/github_release.json'): raise FileNotFoundError(f"github_release文件不存在，请检查程序是否运行正常！") - with open('./JSON/github_release.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/github_release.json', 'r', encoding='utf-8') as file: data_release = json.load(file) if not isinstance(data_release, list): raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") - if not os.path.exists('./JSON/github_user.json'): + if not os.path.exists('./resources/JSON/github_user.json'): raise FileNotFoundError(f"github_user文件不存在，请检查程序是否运行正常！") - with open('./JSON/github_user.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/github_user.json', 'r', encoding='utf-8') as file: data_user = json.load(file) if not isinstance(data_user, list): raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") - conn = sqlite3.connect('./db/github.db') + conn = sqlite3.connect('./resources/db/github.db') cursor = conn.cursor() # 提取所需字段并编号 @@ -209,7 +209,7 @@ def insert_data(): conn.close() def select_articles(e_hour): - conn = sqlite3.connect('./db/github.db') + conn = sqlite3.connect('./resources/db/github.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -279,14 +279,14 @@ def select_articles(e_hour): return result_1, result_2, result_3, result_4 def clear_table(): - conn = sqlite3.connect('./db/github.db') + conn = sqlite3.connect('./resources/db/github.db') cursor = conn.cursor() cursor.execute('DELETE FROM repos') cursor.execute('DELETE FROM releases') conn.commit() conn.close() -def record_md(result, filename="./history/github.md"): +def record_md(result, filename="./resources/history/github.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -376,7 +376,7 @@ def get_filtered_articles(entries, Is_short, choice): def Src_github(e_hour, Is_short): - if not os.path.exists('./db/github.db'): + if not os.path.exists('./resources/db/github.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/qianxin.py b/GotoSend/qianxin.py index c1b1941..21b7c17 100644 --- a/GotoSend/qianxin.py +++ b/GotoSend/qianxin.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/qianxin.db') + conn = sqlite3.connect('./resources/db/qianxin.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -20,7 +20,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/qianxin.db') + conn = sqlite3.connect('./resources/db/qianxin.db') cursor = conn.cursor() for entry in data: cursor.execute(''' @@ -32,11 +32,11 @@ def insert_data(data): def get_qianxin_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/qianxin.json'): + if not os.path.exists('./resources/JSON/qianxin.json'): raise FileNotFoundError(f"qianxin.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/qianxin.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/qianxin.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -59,7 +59,7 @@ def get_qianxin_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/qianxin.db') + conn = sqlite3.connect('./resources/db/qianxin.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -78,13 +78,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/qianxin.db') + conn = sqlite3.connect('./resources/db/qianxin.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/tech_passage.md"): +def record_md(result, filename="./resources/history/tech_passage.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -123,7 +123,7 @@ def get_filtered_articles(entries, Is_short): def Src_qianxin(e_hour, Is_short): - if not os.path.exists('./db/qianxin.db'): + if not os.path.exists('./resources/db/qianxin.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/seebug.py b/GotoSend/seebug.py index 67657ce..1b3e6a5 100644 --- a/GotoSend/seebug.py +++ b/GotoSend/seebug.py @@ -8,7 +8,7 @@ import email.utils def create_database(): - conn = sqlite3.connect('./db/seebug.db') + conn = sqlite3.connect('./resources/db/seebug.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -23,7 +23,7 @@ def create_database(): def insert_data(data): - conn = sqlite3.connect('./db/seebug.db') + conn = sqlite3.connect('./resources/db/seebug.db') cursor = conn.cursor() for entry in data: # 解析并格式化时间 @@ -39,11 +39,11 @@ def insert_data(data): def get_seebug_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/seebug.json'): + if not os.path.exists('./resources/JSON/seebug.json'): raise FileNotFoundError(f"seebug.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/seebug.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/seebug.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -66,7 +66,7 @@ def get_seebug_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/seebug.db') + conn = sqlite3.connect('./resources/db/seebug.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -85,13 +85,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/seebug.db') + conn = sqlite3.connect('./resources/db/seebug.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/sec_news.md"): +def record_md(result, filename="./resources/history/sec_news.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -130,7 +130,7 @@ def get_filtered_articles(entries, Is_short): def Src_seebug(e_hour, Is_short): - if not os.path.exists('./db/seebug.db'): + if not os.path.exists('./resources/db/seebug.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/sougou_wx.py b/GotoSend/sougou_wx.py index 11ddf2f..cec78be 100644 --- a/GotoSend/sougou_wx.py +++ b/GotoSend/sougou_wx.py @@ -4,14 +4,14 @@ import sqlite3 from datetime import datetime, timedelta def clear_table(): - conn = sqlite3.connect('./db/sougou-wx.db') + conn = sqlite3.connect('./resources/db/sougou-wx.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() def create_database(): - conn = sqlite3.connect('./db/sougou-wx.db') + conn = sqlite3.connect('./resources/db/sougou-wx.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -27,7 +27,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/sougou-wx.db') + conn = sqlite3.connect('./resources/db/sougou-wx.db') cursor = conn.cursor() for entry in data: # 检查是否存在相同 title 和 author 的记录 @@ -47,11 +47,11 @@ def insert_data(data): def get_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/sougou-wx.json'): + if not os.path.exists('./resources/JSON/sougou-wx.json'): raise FileNotFoundError(f"sougou-wx.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/sougou-wx.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/sougou-wx.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个关键词的字典 @@ -77,7 +77,7 @@ def get_json(): return total_data def select_articles(): - conn = sqlite3.connect('./db/sougou-wx.db') + conn = sqlite3.connect('./resources/db/sougou-wx.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -124,7 +124,7 @@ def select_articles(): return results -def record_md(result, filename="./history/wx_news.md"): +def record_md(result, filename="./resources/history/wx_news.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -163,7 +163,7 @@ def get_filtered_articles(entries, Is_short): return result def Src_sougou_wx(Is_short): - if not os.path.exists('./db/sougou-wx.db'): + if not os.path.exists('./resources/db/sougou-wx.db'): # 创建数据库和表 create_database() diff --git a/GotoSend/xianzhi.py b/GotoSend/xianzhi.py index 7b495c5..5149be5 100644 --- a/GotoSend/xianzhi.py +++ b/GotoSend/xianzhi.py @@ -6,7 +6,7 @@ import os from datetime import datetime, timedelta def create_database(): - conn = sqlite3.connect('./db/xianzhi.db') + conn = sqlite3.connect('./resources/db/xianzhi.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS articles ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -18,7 +18,7 @@ def create_database(): conn.close() def insert_data(data): - conn = sqlite3.connect('./db/xianzhi.db') + conn = sqlite3.connect('./resources/db/xianzhi.db') cursor = conn.cursor() for entry in data: try: @@ -40,11 +40,11 @@ def insert_data(data): def get_xianzhi_json(): # 检查文件是否存在 - if not os.path.exists('./JSON/xianzhi.json'): + if not os.path.exists('./resources/JSON/xianzhi.json'): raise FileNotFoundError(f"xianzhi.json文件不存在，请检查程序是否运行正常！") # 打开并读取JSON文件 - with open('./JSON/xianzhi.json', 'r', encoding='utf-8') as file: + with open('./resources/JSON/xianzhi.json', 'r', encoding='utf-8') as file: data = json.load(file) # 假设data是一个包含多个JSON对象的列表 @@ -65,7 +65,7 @@ def get_xianzhi_json(): return total_data def select_articles(e_hour): - conn = sqlite3.connect('./db/xianzhi.db') + conn = sqlite3.connect('./resources/db/xianzhi.db') cursor = conn.cursor() # 获取当前日期和时间 @@ -84,13 +84,13 @@ def select_articles(e_hour): return results def clear_table(): - conn = sqlite3.connect('./db/xianzhi.db') + conn = sqlite3.connect('./resources/db/xianzhi.db') cursor = conn.cursor() cursor.execute('DELETE FROM articles') conn.commit() conn.close() -def record_md(result, filename="./history/tech_passage.md"): +def record_md(result, filename="./resources/history/tech_passage.md"): # 读取现有内容 if os.path.exists(filename): with open(filename, 'r', encoding='utf-8') as file: @@ -126,7 +126,7 @@ def get_filtered_articles(entries, Is_short): def Src_xianzhi(e_hour, Is_short): - if not os.path.exists('./db/xianzhi.db'): + if not os.path.exists('./resources/db/xianzhi.db'): # 创建数据库和表 create_database() diff --git a/UpdateLOG.md b/UpdateLOG.md index c04cf23..b27660a 100644 --- a/UpdateLOG.md +++ b/UpdateLOG.md @@ -29,4 +29,5 @@ - 2024年12月25日晚：优化了推送报文格式
- 2024年12月30日：添加并完善了基于搜狗搜索的微信公众号文章监测
- 2024年12月30日晚：为洞见微信资讯推送添加了关键词筛选
-- 2025年01月02日：更新并上线了Github项目监测功能，readme后续更新中
\ No newline at end of file +- 2025年01月02日：更新并上线了Github项目监测功能，readme后续更新中
+- 2025年01月02日晚：再次优化了文件结构，提高可读性
\ No newline at end of file diff --git a/JSON/xianzhi.json b/resources/JSON/xianzhi.json similarity index 100% rename from JSON/xianzhi.json rename to resources/JSON/xianzhi.json diff --git a/db/Format_to_txt.py b/resources/db/Format_to_txt.py similarity index 100% rename from db/Format_to_txt.py rename to resources/db/Format_to_txt.py diff --git a/history/github.md b/resources/history/github.md similarity index 100% rename from history/github.md rename to resources/history/github.md diff --git a/history/sec_news.md b/resources/history/sec_news.md similarity index 100% rename from history/sec_news.md rename to resources/history/sec_news.md diff --git a/history/tech_passage.md b/resources/history/tech_passage.md similarity index 100% rename from history/tech_passage.md rename to resources/history/tech_passage.md diff --git a/history/wx_news.md b/resources/history/wx_news.md similarity index 100% rename from history/wx_news.md rename to resources/history/wx_news.md diff --git a/imgs/add_bot.jpg b/resources/imgs/add_bot.jpg similarity index 100% rename from imgs/add_bot.jpg rename to resources/imgs/add_bot.jpg diff --git a/imgs/bot_config.jpg b/resources/imgs/bot_config.jpg similarity index 100% rename from imgs/bot_config.jpg rename to resources/imgs/bot_config.jpg diff --git a/imgs/config.jpg b/resources/imgs/config.jpg similarity index 100% rename from imgs/config.jpg rename to resources/imgs/config.jpg diff --git a/imgs/github-token.png b/resources/imgs/github-token.png similarity index 100% rename from imgs/github-token.png rename to resources/imgs/github-token.png diff --git a/imgs/group.jpg b/resources/imgs/group.jpg similarity index 100% rename from imgs/group.jpg rename to resources/imgs/group.jpg diff --git a/imgs/result.jpg b/resources/imgs/result.jpg similarity index 100% rename from imgs/result.jpg rename to resources/imgs/result.jpg diff --git a/imgs/run.jpg b/resources/imgs/run.jpg similarity index 100% rename from imgs/run.jpg rename to resources/imgs/run.jpg diff --git a/imgs/start.jpg b/resources/imgs/start.jpg similarity index 100% rename from imgs/start.jpg rename to resources/imgs/start.jpg diff --git a/log/app.log b/resources/log/app.log similarity index 100% rename from log/app.log rename to resources/log/app.log diff --git a/log/core.log b/resources/log/core.log similarity index 100% rename from log/core.log rename to resources/log/core.log diff --git a/spider/common.py b/spider/common.py index 3fa897d..16b750d 100644 --- a/spider/common.py +++ b/spider/common.py @@ -73,11 +73,11 @@ def seebug_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/seebug.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/seebug.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/seebug.json') - logger.info("数据已保存到 ./JSON/seebug.json！") + save_to_json(items, './resources/JSON/seebug.json') + logger.info("数据已保存到 ./resources/JSON/seebug.json！") # 4hou 爬虫 def M_4hou_main(): @@ -91,11 +91,11 @@ def M_4hou_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/4hou.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/4hou.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/4hou.json') - logger.info("数据已保存到 ./JSON/4hou.json！") + save_to_json(items, './resources/JSON/4hou.json') + logger.info("数据已保存到 ./resources/JSON/4hou.json！") # 安全客 爬虫 def anquanke_main(): @@ -109,11 +109,11 @@ def anquanke_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/anquanke.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/anquanke.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/anquanke.json') - logger.info("数据已保存到 ./JSON/anquanke.json！") + save_to_json(items, './resources/JSON/anquanke.json') + logger.info("数据已保存到 ./resources/JSON/anquanke.json！") # sec_wiki 爬虫 def sec_wiki_main(): @@ -127,11 +127,11 @@ def sec_wiki_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/sec_wiki.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/sec_wiki.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/sec_wiki.json') - logger.info("数据已保存到 ./JSON/sec_wiki.json！") + save_to_json(items, './resources/JSON/sec_wiki.json') + logger.info("数据已保存到 ./resources/JSON/sec_wiki.json！") # 华为 爬虫 def huawei_main(): @@ -145,11 +145,11 @@ def huawei_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/huawei.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/huawei.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/huawei.json') - logger.info("数据已保存到 ./JSON/huawei.json！") + save_to_json(items, './resources/JSON/huawei.json') + logger.info("数据已保存到 ./resources/JSON/huawei.json！") # 洞见微信聚合爬虫 def doonsec_main(): @@ -163,11 +163,11 @@ def doonsec_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/doonsec.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/doonsec.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/doonsec.json') - logger.info("数据已保存到 ./JSON/doonsec.json！") + save_to_json(items, './resources/JSON/doonsec.json') + logger.info("数据已保存到 ./resources/JSON/doonsec.json！") # 奇安信攻防社区 爬虫 def qianxin_main(): @@ -181,11 +181,11 @@ def qianxin_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/qianxin.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/qianxin.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/qianxin.json') - logger.info("数据已保存到 ./JSON/qianxin.json！") + save_to_json(items, './resources/JSON/qianxin.json') + logger.info("数据已保存到 ./resources/JSON/qianxin.json！") def run(): seebug_main() diff --git a/spider/freebuf.py b/spider/freebuf.py index 94ac35d..c3612b5 100644 --- a/spider/freebuf.py +++ b/spider/freebuf.py @@ -62,11 +62,11 @@ def freebuf_main(): items = parse_rss(rss_content) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/freebuf.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/freebuf.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - save_to_json(items, './JSON/freebuf.json') - logger.info("数据已保存到 ./JSON/freebuf.json！") + save_to_json(items, './resources/JSON/freebuf.json') + logger.info("数据已保存到 ./resources/JSON/freebuf.json！") except Exception as e: logger.error(f"解析或保存Freebuf RSS内容时发生错误: {e}") diff --git a/spider/github.py b/spider/github.py index 1dded87..648bf13 100644 --- a/spider/github.py +++ b/spider/github.py @@ -85,7 +85,7 @@ def github_main_keyword(key): time.sleep(5) # 将所有结果写入JSON文件 - save_to_json(all_results, './JSON/github_keyword.json') + save_to_json(all_results, './resources/JSON/github_keyword.json') def github_main_repo(key): @@ -128,7 +128,7 @@ def github_main_repo(key): time.sleep(5) # 将所有结果写入JSON文件 - save_to_json(all_results, './JSON/github_repo.json') + save_to_json(all_results, './resources/JSON/github_repo.json') def github_main_release(key): all_results = [] # 用于存储所有结果 @@ -160,7 +160,7 @@ def github_main_release(key): time.sleep(5) # 将所有结果写入JSON文件 - save_to_json(all_results, './JSON/github_release.json') + save_to_json(all_results, './resources/JSON/github_release.json') def github_main_user(key): all_results = [] # 用于存储所有结果 @@ -207,7 +207,7 @@ def github_main_user(key): time.sleep(5) # 将所有结果写入JSON文件 - save_to_json(all_results, './JSON/github_user.json') + save_to_json(all_results, './resources/JSON/github_user.json') def github_main(): if keyword_list: diff --git a/spider/sougou_wx.py b/spider/sougou_wx.py index 8131ae8..4064203 100644 --- a/spider/sougou_wx.py +++ b/spider/sougou_wx.py @@ -108,10 +108,10 @@ def sougou_wx_main(keywords): # print(json_results) # 确保目录存在 - os.makedirs(os.path.dirname('./JSON/sougou-wx.json'), exist_ok=True) + os.makedirs(os.path.dirname('./resources/JSON/sougou-wx.json'), exist_ok=True) # 将解析后的数据保存到 JSON 文件 - with open('./JSON/sougou-wx.json', 'w', encoding='utf-8') as f: + with open('./resources/JSON/sougou-wx.json', 'w', encoding='utf-8') as f: f.write(json_results) if __name__ == "__main__": diff --git a/spider/xianzhi.py b/spider/xianzhi.py index 3be8de7..4bfcddd 100644 --- a/spider/xianzhi.py +++ b/spider/xianzhi.py @@ -61,10 +61,10 @@ def xianzhi_main(): json_data = json.dumps(entries_json, ensure_ascii=False, indent=4) # 保存 JSON 数据到文件 - with open('./JSON/xianzhi.json', 'w', encoding='utf-8') as json_file: + with open('./resources/JSON/xianzhi.json', 'w', encoding='utf-8') as json_file: json_file.write(json_data) - logger.info("数据已保存到 ./JSON/xianzhi.json！") + logger.info("数据已保存到 ./resources/JSON/xianzhi.json！") except Exception as e: logger.error(f"解析或保存先知社区RSS内容时发生错误: {e}") diff --git a/web/app.py b/web/app.py index 58966b3..c5b38d8 100644 --- a/web/app.py +++ b/web/app.py @@ -7,10 +7,10 @@ app = Flask(__name__) # 配置文件路径 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.dirname(BASE_DIR) # 上一个文件夹 -SEC_NEWS_PATH = os.path.join(PARENT_DIR, 'history', 'sec_news.md') -TECH_PASSAGE_PATH = os.path.join(PARENT_DIR, 'history', 'tech_passage.md') -CORE_LOG_PATH = os.path.join(PARENT_DIR, 'log', 'core.log') -WEB_LOG_PATH = os.path.join(PARENT_DIR, 'log', 'app.log') +SEC_NEWS_PATH = os.path.join(PARENT_DIR, 'resources', 'history', 'sec_news.md') +TECH_PASSAGE_PATH = os.path.join(PARENT_DIR, 'resources', 'history', 'tech_passage.md') +CORE_LOG_PATH = os.path.join(PARENT_DIR, 'resources', 'log', 'core.log') +WEB_LOG_PATH = os.path.join(PARENT_DIR, 'resources', 'log', 'app.log') # 配置日志记录器 logging.basicConfig( diff --git a/功能解释.md b/功能解释.md index c433b89..e47e7d8 100644 --- a/功能解释.md +++ b/功能解释.md @@ -5,20 +5,20 @@ config.yaml: 配置各模块参数以及Token密钥 check_config.py: 核查配置信息并获取到配置信息返回给核心程序 -- ./log +- ./resources/log app.py: web运行日志 core.py: 程序运行日志 - ./media 爬取RSS源以及信息源的json数据并保存 -- ./JSON +- ./resources/JSON 存储获取到的原始json数据 - ./GotoSend 对获取到的json数据进行处理，将其存储值db文件内 -- ./db +- ./resources/db 存储处理过后的数据 - ./SendCore @@ -27,8 +27,8 @@ FeishuSendBot.py: 飞书推送核心程序 MailSendBot.py: 邮件推送核心程序 QiweiSendBot.py: 企业微信推送核心程序 -- ./history +- ./resources/history 存储历史推送记录为markdown文件 -- ./Web +- ./web 网页运行程序 \ No newline at end of file