From 1356dcc25c4d99f7402a780fc9c82509df5972a9 Mon Sep 17 00:00:00 2001 From: MasonLiu <2857911564@qq.com> Date: Thu, 2 Jan 2025 13:00:43 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90github=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=E9=83=A8=E5=88=86=E7=9A=84=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Core.py | 122 +++--- Dev_test.py | 56 ++- GotoSend/__pycache__/doonsec.cpython-312.pyc | Bin 6823 -> 7525 bytes GotoSend/__pycache__/github.cpython-312.pyc | Bin 0 -> 144 bytes GotoSend/__pycache__/qianxin.cpython-312.pyc | Bin 6483 -> 6489 bytes .../__pycache__/sougou_wx.cpython-312.pyc | Bin 0 -> 7787 bytes GotoSend/doonsec.py | 37 +- GotoSend/freebuf.py | 2 +- GotoSend/github.py | 369 ++++++++++++++++++ GotoSend/qianxin.py | 2 +- GotoSend/{sougou-wx.py => sougou_wx.py} | 2 +- JSON/xianzhi.json | 300 +++++++------- UpdateLOG.md | 4 +- .../__pycache__/check_config.cpython-312.pyc | Bin 3029 -> 3556 bytes config/check_config.py | 16 +- config/config.yaml | 15 +- config/github_config.yaml | 45 ++- config/keywords.yaml | 4 +- history/github.md | 0 history/sec_news.md | 10 - history/tech_passage.md | 228 ----------- media/github.py | 18 - .../__pycache__/common.cpython-312.pyc | Bin .../__pycache__/freebuf.cpython-312.pyc | Bin spider/__pycache__/github.cpython-312.pyc | Bin 0 -> 960 bytes spider/__pycache__/sougou_wx.cpython-312.pyc | Bin 0 -> 5854 bytes .../__pycache__/xianzhi.cpython-312.pyc | Bin 3371 -> 3376 bytes {media => spider}/common.py | 0 {media => spider}/freebuf.py | 0 spider/github.py | 189 +++++++++ media/sougou-wx.py => spider/sougou_wx.py | 5 +- {media => spider}/xianzhi.py | 0 32 files changed, 923 insertions(+), 501 deletions(-) create mode 100644 GotoSend/__pycache__/github.cpython-312.pyc create mode 100644 GotoSend/__pycache__/sougou_wx.cpython-312.pyc rename GotoSend/{sougou-wx.py => sougou_wx.py} (98%) create mode 100644 history/github.md delete mode 100644 media/github.py rename {media => spider}/__pycache__/common.cpython-312.pyc (100%) rename {media => spider}/__pycache__/freebuf.cpython-312.pyc (100%) create mode 100644 spider/__pycache__/github.cpython-312.pyc create mode 100644 spider/__pycache__/sougou_wx.cpython-312.pyc rename {media => spider}/__pycache__/xianzhi.cpython-312.pyc (87%) rename {media => spider}/common.py (100%) rename {media => spider}/freebuf.py (100%) create mode 100644 spider/github.py rename media/sougou-wx.py => spider/sougou_wx.py (95%) rename {media => spider}/xianzhi.py (100%) diff --git a/Core.py b/Core.py index 2975592..df8e041 100644 --- a/Core.py +++ b/Core.py @@ -14,9 +14,11 @@ import requests from datetime import datetime, timedelta from SendCore.FeishuSendBot import SendToFeishu, gen_sign from SendCore.QiweiSendBot import SendToWX -from media.common import run, seebug_main, M_4hou_main, anquanke_main, sec_wiki_main, huawei_main, doonsec_main, qianxin_main -from media.freebuf import freebuf_main -from media.xianzhi import xianzhi_main +from spider.common import run, seebug_main, M_4hou_main, anquanke_main, sec_wiki_main, huawei_main, doonsec_main, qianxin_main +from spider.freebuf import freebuf_main +from spider.xianzhi import xianzhi_main +from spider.sougou_wx import sougou_wx_main +from spider.github import github_main from GotoSend.M_4hou import Src_4hou from GotoSend.anquanke import Src_anquanke from GotoSend.doonsec import Src_doonsec @@ -24,7 +26,9 @@ from GotoSend.xianzhi import Src_xianzhi from GotoSend.freebuf import Src_freebuf from GotoSend.qianxin import Src_qianxin from GotoSend.seebug import Src_seebug -from config.check_config import get_core_config, get_debug_config +from GotoSend.sougou_wx import Src_sougou_wx +from GotoSend.github import Src_github +from config.check_config import get_core_config, get_debug_config, get_kewords_config from loguru import logger # 清除所有已有的日志记录器配置 @@ -49,7 +53,10 @@ def signal_handler(sig, frame): signal.signal(signal.SIGINT, signal_handler) # Ctrl+C signal.signal(signal.SIGTERM, signal_handler) # kill命令 webhook_url_once, timestamp_once, sign_once = gen_sign() -e_hour, choice, fs_activate, wx_activate, ding_activate, lx_activate, url_web = get_core_config() +e_hour, time_choice, choice, fs_activate, wx_activate, ding_activate, lx_activate, url_web = get_core_config() +Sogou_WX, Doonsec_switch, Doonsec = get_kewords_config() + + def check_avaliable(info_long, info_short, title, webhook_url, timestamp, sign): if info_long: # 发送完整文章相关内容 @@ -80,10 +87,8 @@ def check_avaliable(info_long, info_short, title, webhook_url, timestamp, sign): if not info_long and not info_short: logger.info(f"{title}数据为空，跳过执行。") -def send_job(time_1): +def send_job_RSS(time_1): # 爬取数据 - logger.info(f"发送程序启动，当前时间为：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - logger.info("正在启动各爬虫并获取资源中...") seebug_main() anquanke_main() huawei_main() @@ -96,7 +101,7 @@ def send_job(time_1): # 分析各个数据源的结果（输出长结果） result_4hou_long = Src_4hou(time_1, False) result_anquanke_long = Src_anquanke(time_1, False) - result_doonsec_long = Src_doonsec(time_1, False) + result_doonsec_long = Src_doonsec(time_1, False, Doonsec_switch, Doonsec) result_xianzhi_long = Src_xianzhi(time_1, False) result_freebuf_long = Src_freebuf(time_1, False) result_qianxin_long = Src_qianxin(time_1, False) @@ -104,7 +109,7 @@ def send_job(time_1): # 分析各个数据源的结果（输出短结果） result_4hou_short = Src_4hou(time_1, True) result_anquanke_short = Src_anquanke(time_1, True) - result_doonsec_short = Src_doonsec(time_1, True) + result_doonsec_short = Src_doonsec(time_1, True, Doonsec_switch, Doonsec) result_xianzhi_short = Src_xianzhi(time_1, True) result_freebuf_short = Src_freebuf(time_1, True) result_qianxin_short = Src_qianxin(time_1, True) @@ -120,17 +125,21 @@ def send_job(time_1): check_avaliable(result_qianxin_long, result_qianxin_short, "奇安信攻防社区资讯", webhook_url, timestamp, sign) check_avaliable(result_seebug_long, result_seebug_short, "Seebug社区资讯", webhook_url, timestamp, sign) - if fs_activate == "True": - send_result = SendToFeishu(f"[点此访问]({url_web})网站以查看全部文章。", "单次运行结束", webhook_url, timestamp, sign) - logger.info(send_result) - else: - pass - if wx_activate == "True": - send_result = SendToWX(f"[点此访问]({url_web})网站以查看全部文章。", "单次运行结束") - logger.info(send_result) - else: - pass - logger.info("执行完毕，等待下一次执行...") +def send_job_SX(): + sougou_wx_main(Sogou_WX) + result_sx_long = Src_sougou_wx(False) + result_sx_short = Src_sougou_wx(True) + webhook_url, timestamp, sign = gen_sign() + check_avaliable(result_sx_long, result_sx_short, "微信公众号关键词相关内容", webhook_url, timestamp, sign) + +def send_job_github(time_1): + github_main() + result_github_1_long, result_github_2_long, result_github_3_long = Src_github(time_1, False) + result_github_1_short, result_github_2_short, result_github_3_short = Src_github(time_1, True) + webhook_url, timestamp, sign = gen_sign() + check_avaliable(result_github_1_long, result_github_1_short, "Github项目监控-关键词监控", webhook_url, timestamp, sign) + check_avaliable(result_github_2_long, result_github_2_short, "Github项目监控-项目更新情况", webhook_url, timestamp, sign) + check_avaliable(result_github_3_long, result_github_3_short, "Github项目监控-大佬工具", webhook_url, timestamp, sign) # 探测rss源状态 def check_rss_status(url): @@ -167,12 +176,22 @@ def test_rss_source(): return rss_info -def main_loop(choice): - if choice == 1: +def main_job(e_hour): + logger.info(f"发送程序启动，当前时间为：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + logger.info("正在启动各爬虫并获取资源中...") + if 0 in choice: + send_job_RSS(e_hour) + if 1 in choice: + send_job_SX() + if 2 in choice: + send_job_github(e_hour) + +def main_loop(time_choice): + if time_choice == 1: while True: try: # 执行任务 - send_job(e_hour) + main_job(e_hour) time.sleep(e_hour * 60 * 60 - 3 * 60) except Exception as e: @@ -181,13 +200,13 @@ def main_loop(choice): # logger.info(result) exit() - elif choice == 0: + elif time_choice == 0: # 设置每天的特定时间点执行job函数 - schedule.every().day.at("09:00").do(send_job, 12) - schedule.every().day.at("12:00").do(send_job, 3) - schedule.every().day.at("15:00").do(send_job, 3) - schedule.every().day.at("18:00").do(send_job, 3) - schedule.every().day.at("21:00").do(send_job, 3) + schedule.every().day.at("09:00").do(main_job, 12) + schedule.every().day.at("12:00").do(main_job, 3) + schedule.every().day.at("15:00").do(main_job, 3) + schedule.every().day.at("18:00").do(main_job, 3) + schedule.every().day.at("21:00").do(main_job, 3) while True: schedule.run_pending() @@ -198,26 +217,31 @@ def send_first_message(): start_info = "" start_info += "程序已启动，当前时间为：" + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n" start_info += "程序作者：MasonLiu \t 开源地址：[GM-gitea](https://git.masonliu.com/MasonLiu/PyBot)" + "\n" - if choice == 1: + if time_choice == 1: start_info += "时间配置：每隔" + str(e_hour) + "小时执行一次推送\n" - else: + elif time_choice == 0: start_info += "时间配置：每天固定时间点执行推送\n" - if fs_activate == "True": - result = SendToFeishu(start_info, "程序信息", webhook_url_once, timestamp_once, sign_once) - logger.info(result) - result = SendToFeishu(rss_info, "RSS源状态", webhook_url_once, timestamp_once, sign_once) - # logger.info(rss_info) - logger.info(result) - else: - pass - if wx_activate == "True": - result = SendToWX(start_info, "程序信息") - logger.info(result) - result = SendToWX(rss_info, "RSS源状态") - # logger.info(rss_info) - logger.info(result) - else: - pass + if 0 in choice: + if fs_activate == "True": + result = SendToFeishu(start_info, "程序信息", webhook_url_once, timestamp_once, sign_once) + logger.info(result) + result = SendToFeishu(rss_info, "RSS源状态", webhook_url_once, timestamp_once, sign_once) + # logger.info(rss_info) + logger.info(result) + send_result = SendToFeishu(f"[点此访问]({url_web})网站以查看全部文章。", "首次运行提醒", webhook_url_once, timestamp_once, sign_once) + logger.info(send_result) + else: + pass + if wx_activate == "True": + result = SendToWX(start_info, "程序信息") + logger.info(result) + result = SendToWX(rss_info, "RSS源状态") + # logger.info(rss_info) + logger.info(result) + send_result = SendToWX(f"[点此访问]({url_web})网站以查看全部文章。", "首次运行提醒") + logger.info(send_result) + else: + pass if __name__ == "__main__": logger.info("程序正在运行当中。") @@ -228,4 +252,4 @@ if __name__ == "__main__": # 主程序 send_first_message() - main_loop(choice) \ No newline at end of file + main_loop(time_choice) \ No newline at end of file diff --git a/Dev_test.py b/Dev_test.py index cdb73d4..a68e53d 100644 --- a/Dev_test.py +++ b/Dev_test.py @@ -1,10 +1,54 @@ +# -*- coding: utf-8 -*- +""" +@Author: MasonLiu +@Description: 本程序可以爬取各安全资讯源，并发送到飞书群组。 +""" + +import schedule +import os +import signal +import sys +import time +import yaml +import requests +from datetime import datetime, timedelta from SendCore.FeishuSendBot import SendToFeishu, gen_sign +from SendCore.QiweiSendBot import SendToWX +from media.common import run, seebug_main, M_4hou_main, anquanke_main, sec_wiki_main, huawei_main, doonsec_main, qianxin_main +from media.freebuf import freebuf_main +from media.xianzhi import xianzhi_main +from media.sougou_wx import sougou_wx_main +from media.github import github_main +from GotoSend.M_4hou import Src_4hou +from GotoSend.anquanke import Src_anquanke +from GotoSend.doonsec import Src_doonsec +from GotoSend.xianzhi import Src_xianzhi +from GotoSend.freebuf import Src_freebuf +from GotoSend.qianxin import Src_qianxin +from GotoSend.seebug import Src_seebug +from GotoSend.sougou_wx import Src_sougou_wx +# from GotoSend.github import Src_github +from config.check_config import get_core_config, get_debug_config, get_kewords_config +from loguru import logger -webhook_url, timestamp, sign = gen_sign() +# # 全局变量 +# webhook_url_once, timestamp_once, sign_once = gen_sign() +# e_hour, time_choice, choice, fs_activate, wx_activate, ding_activate, lx_activate, url_web = get_core_config() +# Sogou_WX = get_kewords_config() +# webhook_url, timestamp, sign = gen_sign() -# 测试用消息体 -test_msg = { - "请单件文档查看昨天讨论的方案相关飞书文档，注意作者为 **张三** 版本为 \n*002* ,版本 ~~001~~ 已经删除。文件地址是 [https://www.feishu.cn](https://www.feishu.cn),打开次数:1次" - } +# sougou_wx_main(Sogou_WX) +# result_sx_long = Src_sougou_wx(False) +# result_sx_short = Src_sougou_wx(True) +# webhook_url, timestamp, sign = gen_sign() +# SendToFeishu(result_sx_long, "微信公众号关键词相关内容", webhook_url, timestamp, sign) -SendToFeishu(test_msg, "先知社区资讯递送", webhook_url, timestamp, sign) \ No newline at end of file +github_main() + + +# # 测试用消息体 +# test_msg = { +# "请单件文档查看昨天讨论的方案相关飞书文档，注意作者为 **张三** 版本为 \n*002* ,版本 ~~001~~ 已经删除。文件地址是 [https://www.feishu.cn](https://www.feishu.cn),打开次数:1次" +# } + +# SendToFeishu(test_msg, "先知社区资讯递送", webhook_url, timestamp, sign) diff --git a/GotoSend/__pycache__/doonsec.cpython-312.pyc b/GotoSend/__pycache__/doonsec.cpython-312.pyc index 67cfc5adbbfd64c098329a05fe75d5a04890054e..95d06a5f09ceff28c80cd51bf06fc8f456e602fe 100644 GIT binary patch delta 1374 zcmaJ>O>7%Q6rSWK1(?WGy@NfzU2*It_Ka2A)4>j-Czo?FuuA#mw zQ$~6&GM&w)BNwu|ab+?;o6SY04Rc#$G||u06p=p_MdK%D9QZC1VHk_SANm{cPmBmz zd`Av5oWM1!QBw@~56Jj}PLsvNfYPcZiVnO`x>6@P>OgajZJ@P-n`J4$LEF8Va~`U=Hux!Qm9JBb{a9N(^dzw z*^Fr5-BNP*&+J@L5%fAgLE zdVcfyP2ZQ^`|oVE?1s9_{+>0t;$VH^8dp()_t?ItZP(LQYCl`{^gS5e?~m^GM+Zvd z@pAucsXnpheB^I^44`;oZRC;cxfQ(`Ee;fu#Y>y<;>jOmt-_Ell2LS1owt{ZbKeVX z2WrDM>+XQuBw5zt_Nh9LgS$i^+8*`vG0hZeoXs1V99l3;TNpEA=6n|A&ceaH9l+~#J@_Fb;m`fAPOv$;p;fckXH}Y`r*zl~EQiHx9bMT1 zxr|87)j#d!ELf}Hg41&1_P`4-QsL+vbs_5L5g{r|2}uO;VB~CQDnS+A6kCi#4qnX@wS~X-g^Dt+kC^ncWJl zn?s?XHx2LNNkjw>y(kg!B6wDimMDcB6cKM8iuDh0CT;A4d3@jZ=FPnKW_}I4?9YBu z6d5tQ|9$1wU=*RBoU}|j0tfG^+3VS$?%BdcS@&+CqEDBZ_Zy1tTgf3ra{@B_x?fke zh3a!H6WGRwsTi?@)N9FyRp zur0KzV*5kvkOX$Y2PqwQT$4y6VzuTAJ8J({8 z6?eQdf88RP!}~H)qqWDO944deuww0fTbn>87+k1*i42H%0@V0PTmUV8flM>`Zq10V z;@Htpn1B#}agnWqj|fuX^N?;ID+o64vT2TR{E#+p_E1GFwIVtq4cMXo5z)b|p;shF zg17CbGQ2oC$X$jz-aO^yXoL=Pd=p3_+cmugDG&Lk z8JJECiS_syc$(<;&?%;kZsedE%fPdk8`_eUX!DNdy?h+a+vps;NS>2RjPt*0-Gd*= YED7Q8pVJUXrCp&X`MS~x7gOE;01ox8rvLx| diff --git a/GotoSend/__pycache__/github.cpython-312.pyc b/GotoSend/__pycache__/github.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b0510d175175c12ac067d3be6c63cb0c4120684 GIT binary patch literal 144 zcmX@j%ge<81P9+|r-SInAOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdrJrgQ6P%iprW=x< zpQG!XpOP9AP??lpq8pN0ToU7+Uy>i3nwJujo>`Jnnxt1y`HRCQH$SB`C)KWq6{w#P Rh>JmtkIamWj77{q769+ZA`bun literal 0 HcmV?d00001 diff --git a/GotoSend/__pycache__/qianxin.cpython-312.pyc b/GotoSend/__pycache__/qianxin.cpython-312.pyc index 3d3d4d91c5b861b2dba6589472601c81d1ab71aa..f18811a4d7d2e32c6070e868837417432fa2f75b 100644 GIT binary patch delta 157 zcmca?bkm6UG%qg~0}$}X6sK2j+=5lS3=FkQHOwhOlka1Bceha}MW v*$gS1bD81dB9lJ|i8G2$<`A}J6r1cOJcTi2^DSX#MwXIG+|rvBMU7Yh6$&Uq delta 132 zcmca(;{FA$dl(_|}co`UKnQE9*1SkI&6zAoi%L*1@28swxzAGfoDqO>o!aez+ gkoaT)VJk+_$=?M@&dT2ua3ZnHbx0Vw(^<5J$M%SVoq--K{d# z$ibwN!rqv*sm(5Ck`!XIGb0Kncq+4rlVum!;RjXwM~XcaO^2$bYHSC7MIlUO*Yju3 zZ9Q!a$9P<{@rQ@{l zq~~-{8+aq9KUR*AutkM<182BI#4UH3coSzhwgXzU&|>CI$rb_)6Ij9-(-P%*MUP>^ zmUR!Sa@bTd!#O2hat-jZ7S|l_mYh9tJMkJ%7yrJSf$A)h5`#dfUls3=S#w0v@CXW^ zemb))et#(;8X*Eizvkg~PLH4oIfvHx&Q6v z@8EUx|LU^}cv<}av$xG&2OqMdZcnqFZERp2tsSiWmB#jt_SAP}%XMs`xHz`a(P7_b zZ)0CtvhkoowBKj@Cv8sA{o0Ix17WC70yp*$(?F88Wxab8MA}Rs(mC_ zK}$~x0^k8VpaKsKApJkU19tz%!GkJ*>e1nY2#t^d)!IN2AUP7i8Nrpp*Iw171SA18 zs2&F$Cy9ybM`)!pw5k)@HHj;eX$|^m6|3sB)rn3D!Mq*|O3;qK$EuPBH2^K0Bfh(v zQd<%4-*&*Py&Y`rO4MQxzF=>&v+=$9Ec-lL=cs2B+Y2638f04OBCF$V{xhkTq;oVn z+U;!}*g;w|d`Ee!lEwSJlFiuam26^nS0=1gvWfj`dr0%kbhO!FQ| z7#1lI;7P|lM(^@~n@Ebs2&K?e0zdIzpn9ygSaRgU$6r-A9j8ye@rWLVAM62) z?+0|4ez1o*qX)@ly{{S30M-?)8ZJBrG|9OWXhNXS#4iE8RPAUJszo9(q#{r!BH(Z0 z=5ewU4G}hr!=GDSv%kI7kwK^WMX$$K59-nk3yYTi@zlbcNxjJ$8 z%BM>ojxL@5#n+*C7C)a{n!d6)^~<~0rtV&tSo-yci?g%eTsj7dFx3_rp&A6p|c+=SknHB*eBq0x;WF+LnMZh0dD8v=s5H^jha&bzB zwVDOIur`yd@`^Gw=#=_oHGkYCN}_COaJhMhS8DM3Je*w+fJIW?LEa-%czpovyiQK0 zfo=|nvd$#}3`tH;507C`lw{q@PPdPbzoz9qz5!lqZ^upZh)jH4I^afw4A1IoMc;Dp`K_5|)!0LwIr z7WBr+9pgJDpBsNJTv$7=-*($n8s6ADXX*g%=7xEWVGj=&}e?GNzd-UN9F-S6-&(%~iLp z#nb-KmQX=>n-DG$=d4oLAc2YEvf!av%Y4x@_mNIpHr_O5pW1#WuP9njaz6iDesD`@ z+gw3)G{5*fbB+n-g{*V=mC@2o=Z~K|9x~1dbEUhYMWyF=p4%C03<+~Z+oFZ*&zGMo z5AF=L%@sZoE!z04naMSc*<)5@T=%7+Bw{F;=6`S4{B15+iy2|0vCYVk7p8K*ufcP{ zlUx=tuzi+SZzXS9>G~qg3WAM+QXxizaoA0O++TlgN<(lz{)DSa5*SBR0aY4!Io0J9 z(D}ieMo29tDr*oCjtY=#0WL=;!2-Z!4M0w}ub^Q>e+rV*15J&Q3Q$5#fC7L8cN?Jt zG^bE50NJ(2rU|Hfi6lS*Fckw}q5kZ&=Hy+{0uVuBE$GW=M~`nmL193YFpXBqW+apY zprZa%ut>Hbl-4Sx0tCkdH1DAgGETLf5aLEOZ)heQ@#_#VgyI0Z>8~DbQj<*^r;zus zBLm6-6B-u%|D=n1d%RBY$%>VHaz&Row@jZv8rxaNf#zm~V` zHqzQw4@a#%ok_09*4x|nuCUPD*wWa+KADpFEoGfmWdl`Z99#CnuCkV0W$gkc_5Ks7 zE}3&U@FG52rReIKecJ6EY4Yu{P6c28z90COzIE zh@lWP^!SkxC81YItHi0P)9qHMsjRgh@8OlhqRa?hxBG~*=a@_n3NE0g3TPA2#@K;10zZpdVGC(|546DVzk zpSVuJ_DAOw0Cn}*la=F@?^KWOjppXR|8!{Md{I>-w`$Z8qc+pIDdtP(8)7rr;W*_tyobJk5`ns3#}kR4sO z{ytJ0H&2n#5;kZ#_wtl3nw59fbOsnT#pXcNShr#7@Sn7lWpj*%ez7bBX^*DyriqrA z0cp*XhH=A$Dasfob>q4T!yUbK)bZaB%*X%)T}7AR2IIJK!hD-CPHCozDRFGzOQtx& z6#qF!L;Ih>z$zuD4a?$ISfCrax+nLRqTiKj>$Aw;m9MYYlQ-3LeKxwOr$KTvi^TWY z8|$AUd+;7j7Mz(-ke`JG{!0jVAeBWE%F}2cAb?zB9LKt$0JR2;f7~&5gj`LiJt~fZ z12sB9`auH_<`kU=?erSj@n&9C0@S5MEQ6%xD#AYeFnQdO#I}{BayWr`Hnl_802YLe zxY~rPEl|nCkWDSv@M8t8@b?g^a03Mi4&Y5g0x%gHEvw_+04G3TLLMj3q#d|dmO{dS zJ9a~QPJ(lvtoQIk$p;Z<9u&`x6CbU>_dw>Qdw>%qZ9oFFTW2^-GltI%zxT7Sp%{pccB>+XlU?!pFq4l@cm?62 ztr6?i(I$l|jrC2F!K3rqYD^;7nazo7m)l;m<2+QL$A}2nkGBV|X z6!*f3sdi+`cPN{@K{?pKN$KlL!Edhq{8gb0mfEj6Tz(J3i4XgpO}8{TdH323G~dJD zbD7ZjF#B4V{o94B!KJgG+&lYQ+=e5M4}bK9e|!J#=-XI?_qY!qxKk%5ELGB`6}qY) zS#%xCgw0Rm{&B;bps?v7`!M;csK^{nMFrUL;aCQ8Wi)XK&QDfUqz0o1ZG_@*;Kv%A zn{n}?%rrv66OyBn@FKi`{TsVx{K&G(ZFBK?1^iy(1)fU>xYyymzk{FXgS|hBqDC|1 z$Cb*Gj}INTF*>Txp7MR6FOC|l)4JeOA$ z8!o&M+!bn@p=X@GVM6}d;_Ljax*L7hyKV_#Yg>eA8&gLaQ#fZ!#x$(kBg}S1KGl*j z4Qq9TsZMm?oG}k;ZG@?f%dO=RrhHr-+W;&2U?ZYUW4-fKZd7j>ZFx9NjQ6L4N56%e zza%DRB%yTasC5an;XaAL#;guY_-vq@n3eRQ!motaL1v?b*WeST%v0dot zPQ58`(Bl`l*Go|0%w)!iylP%wy}+!SX9^w|P~Q5#?;3SLq@lB>O!HJJ+!d71<=f!s z3Ve$xxNY4ME`NG<=bW{EOcOQdjhXJ4bCeGzGu_<-PM4>Qmlv*{U7Y+ho{?FY{ru~TpW|H2ryt+Fc4}$-SCHxXcbhZvHvJ1x=XAh$2)mk|2n$koGI2{|cG@3zgreXu@*eWP*yd zK=qRnBJcil)kOApuMoEBXVO;Aj&Z2|AH*|gRyY>sX5~YfE$f*D)-G3 ie_)Jby{C<^?po7W--H>S!=`6{&pa11BdSmt)c*oFDf7<& literal 0 HcmV?d00001 diff --git a/GotoSend/doonsec.py b/GotoSend/doonsec.py index 74b98df..6e30760 100644 --- a/GotoSend/doonsec.py +++ b/GotoSend/doonsec.py @@ -68,20 +68,37 @@ def get_doonsec_json(): return total_data -def select_articles(e_hour): +def select_articles(e_hour, Doonsec_switch, Doonsec): conn = sqlite3.connect('./db/doonsec.db') cursor = conn.cursor() # 获取当前日期和时间 now = datetime.now() - start_time = now - timedelta(hours=e_hour) + start_time = now - timedelta(hours=e_hour, minutes=3) end_time = now - # 查询指定时间段内的数据 - cursor.execute(''' - SELECT * FROM articles - WHERE pubDate BETWEEN ? AND ? - ''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), end_time.strftime('%Y-%m-%d %H:%M:%S'))) + if Doonsec_switch == False: + # 查询指定时间段内的数据 + cursor.execute(''' + SELECT * FROM articles + WHERE pubDate BETWEEN ? AND ? + ''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), end_time.strftime('%Y-%m-%d %H:%M:%S'))) + elif Doonsec_switch == True: + # 查询指定时间段内且title包含特定关键词的数据 + placeholders = ', '.join(['?'] * len(Doonsec)) + query = f''' + SELECT * FROM articles + WHERE pubDate BETWEEN ? AND ? + AND (title LIKE ? OR title LIKE ? OR title LIKE ?) + ''' + + # 构建参数列表 + params = [start_time.strftime('%Y-%m-%d %H:%M:%S'), end_time.strftime('%Y-%m-%d %H:%M:%S')] + for keyword in Doonsec: + params.append(f'%{keyword}%') + + # 执行查询 + cursor.execute(query, params) results = cursor.fetchall() conn.close() @@ -133,7 +150,7 @@ def get_filtered_articles(entries, Is_short): return result -def Src_doonsec(e_hour, Is_short): +def Src_doonsec(e_hour, Is_short, Doonsec_switch, Doonsec): if not os.path.exists('./db/doonsec.db'): # 创建数据库和表 create_database() @@ -148,7 +165,7 @@ def Src_doonsec(e_hour, Is_short): insert_data(M_doonsec_data) # 查询指定时间段内的数据 - filtered_articles = select_articles(e_hour) + filtered_articles = select_articles(e_hour, Doonsec_switch, Doonsec) # print(filtered_articles) if filtered_articles: @@ -158,7 +175,7 @@ def Src_doonsec(e_hour, Is_short): return False if __name__ == "__main__": - reslts = Src_doonsec(4, False) + reslts = Src_doonsec(24, False, True, ["webshell", "2000", "POC"] ) if reslts != False: print(reslts) else: diff --git a/GotoSend/freebuf.py b/GotoSend/freebuf.py index 24a0ec8..35c4c0d 100644 --- a/GotoSend/freebuf.py +++ b/GotoSend/freebuf.py @@ -50,7 +50,7 @@ def get_freebuf_json(): # 假设data是一个包含多个JSON对象的列表 if not isinstance(data, list): - raise ValueError("JSON文件格式错误，请检查common.py是否异常！") + raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") # 提取所需字段并编号 total_data = [] diff --git a/GotoSend/github.py b/GotoSend/github.py index e69de29..83779fa 100644 --- a/GotoSend/github.py +++ b/GotoSend/github.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- + +import json +import sqlite3 +import os +from datetime import datetime, timedelta + +def create_database(): + conn = sqlite3.connect('./db/github.db') + cursor = conn.cursor() + + cursor.executescript(''' + CREATE TABLE IF NOT EXISTS keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT, + link TEXT, + description TEXT, + pubDate DATETIME, + author TEXT, + keyword TEXT, + language TEXT, + is_sended BOOLEAN + ); + CREATE TABLE IF NOT EXISTS repos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT, + link TEXT, + description TEXT, + pubDate DATETIME, + author TEXT, + keyword TEXT, + link2 TEXT + ); + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT, + link TEXT, + description TEXT, + pubDate DATETIME, + author TEXT, + keyword TEXT, + language TEXT, + is_sended BOOLEAN + ); + ''') + + conn.commit() + conn.close() + + +def insert_data(): + # 检查文件是否存在 + # 打开并读取JSON文件 + # 假设data是一个包含多个JSON对象的列表，然后校验JSON格式是否异常 + if not os.path.exists('./JSON/github_keyword.json'): + raise FileNotFoundError(f"github_keyword文件不存在，请检查程序是否运行正常！") + with open('./JSON/github_keyword.json', 'r', encoding='utf-8') as file: + data_keyword = json.load(file) + if not isinstance(data_keyword, list): + raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") + + if not os.path.exists('./JSON/github_repo.json'): + raise FileNotFoundError(f"github_repo文件不存在，请检查程序是否运行正常！") + with open('./JSON/github_repo.json', 'r', encoding='utf-8') as file: + data_repo = json.load(file) + if not isinstance(data_repo, list): + raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") + + if not os.path.exists('./JSON/github_user.json'): + raise FileNotFoundError(f"github_user文件不存在，请检查程序是否运行正常！") + with open('./JSON/github_user.json', 'r', encoding='utf-8') as file: + data_user = json.load(file) + if not isinstance(data_user, list): + raise ValueError("JSON文件格式错误，请检查爬取程序是否异常！") + + conn = sqlite3.connect('./db/github.db') + cursor = conn.cursor() + + # 提取所需字段并编号 + for index, item in enumerate(data_keyword, start=1): + entry = { + "id": index, + "title": item.get("name", ""), + "link": item.get("link", ""), + "description": item.get("description", ""), + "pubDate": item.get("created_at", ""), + "author": item.get("author", ""), + "keyword": item.get("keyword", ""), + "language": item.get("language", "") + } + try: + # 解析 pubDate 字符串为 datetime 对象 + pub_date = datetime.strptime(entry['pubDate'], '%Y-%m-%dT%H:%M:%SZ') + # 格式化 pubDate 为所需的格式 + formatted_pub_date = pub_date.strftime('%Y-%m-%d %H:%M:%S') + except ValueError: + # 如果解析失败，使用原始 pubDate 字符串 + formatted_pub_date = entry['pubDate'] + + + # 检查是否存在相同 title 和 author 的记录 + cursor.execute(''' + SELECT 1 FROM keywords WHERE title = ? AND author = ? + ''', (entry['title'], entry['author'])) + + if cursor.fetchone() is None: + # 如果没有找到相同记录，则插入新记录 + cursor.execute(''' + INSERT INTO keywords (title, link, description, pubDate, author, language, keyword) + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (entry['title'], entry['link'], entry['description'], formatted_pub_date, entry['author'], entry['language'], entry['keyword'])) + + for index, item in enumerate(data_repo, start=1): + entry = { + "id": index, + "title": item.get("name", ""), + "link": item.get("link", ""), + "description": item.get("description", ""), + "pubDate": item.get("updated_at", ""), + "author": item.get("author", ""), + "keyword": item.get("keyword", ""), + "link2": item.get("link_2", "") + } + try: + # 解析 pubDate 字符串为 datetime 对象 + pub_date = datetime.strptime(entry['pubDate'], '%Y-%m-%dT%H:%M:%SZ') + # 格式化 pubDate 为所需的格式 + formatted_pub_date = pub_date.strftime('%Y-%m-%d %H:%M:%S') + except ValueError: + # 如果解析失败，使用原始 pubDate 字符串 + formatted_pub_date = entry['pubDate'] + + cursor.execute(''' + INSERT INTO repos (title, link, description, pubDate, author, link2, keyword) + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (entry['title'], entry['link'], entry['description'], formatted_pub_date, entry['author'], entry['link2'], entry['keyword'])) + + + # 插入 users 数据 + for index, item in enumerate(data_user, start=1): + entry = { + "id": index, + "title": item.get("name", ""), + "link": item.get("link", ""), + "description": item.get("description", ""), + "pubDate": item.get("created_at", ""), + "author": item.get("author", ""), + "keyword": item.get("keyword", ""), + "language": item.get("language", "") + } + try: + # 解析 pubDate 字符串为 datetime 对象 + pub_date = datetime.strptime(entry['pubDate'], '%Y-%m-%dT%H:%M:%SZ') + # 格式化 pubDate 为所需的格式 + formatted_pub_date = pub_date.strftime('%Y-%m-%d %H:%M:%S') + except ValueError: + # 如果解析失败，使用原始 pubDate 字符串 + formatted_pub_date = entry['pubDate'] + + # 检查是否存在相同 title 和 author 的记录 + cursor.execute(''' + SELECT 1 FROM users WHERE title = ? AND author = ? + ''', (entry['title'], entry['author'])) + + if cursor.fetchone() is None: + # 如果没有找到相同记录，则插入新记录 + cursor.execute(''' + INSERT INTO users (title, link, description, pubDate, author, keyword, language) + VALUES (?, ?, ?, ?, ?, ?, ?) + ''', (entry['title'], entry['link'], entry['description'], formatted_pub_date, entry['author'], entry['keyword'], entry['language'])) + + conn.commit() + conn.close() + +def select_articles(e_hour): + conn = sqlite3.connect('./db/github.db') + cursor = conn.cursor() + + # 获取当前日期和时间 + now = datetime.now() + two_months_ago = now - timedelta(days=60) # 假设两个月大约60天 + start_time = now - timedelta(hours=e_hour, minutes=3) + + # 查询指定时间段内的数据 + cursor.execute(''' + SELECT * FROM keywords + WHERE is_sended IS NULL AND pubDate BETWEEN ? AND ? + ORDER BY pubDate DESC + ''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S'))) + + result_1 = cursor.fetchall() + + if result_1: + for row in result_1: + keyword_id = row[0] + cursor.execute(''' + UPDATE keywords + SET is_sended = True + WHERE id = ? + ''', (keyword_id,)) + + conn.commit() + + cursor.execute(''' + SELECT * FROM repos + WHERE pubDate BETWEEN ? AND ? + ''', (start_time.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S'))) + + result_2 = cursor.fetchall() + + # 查询最近的5条未被标记为True的消息且发布时间不超过两个月 + cursor.execute(''' + SELECT * FROM users + WHERE is_sended IS NULL AND pubDate BETWEEN ? AND ? + ORDER BY pubDate DESC + LIMIT 5 + ''', (two_months_ago.strftime('%Y-%m-%d %H:%M:%S'), now.strftime('%Y-%m-%d %H:%M:%S'))) + + # 查询最近的3条未被标记为True的消息 + # cursor.execute(''' + # SELECT * FROM users + # WHERE is_sended IS NULL + # ORDER BY pubDate DESC + # LIMIT 5 + # ''') + + result_3 = cursor.fetchall() + # print(results) + + if result_3: + for row in result_3: + user_id = row[0] + cursor.execute(''' + UPDATE users + SET is_sended = True + WHERE id = ? + ''', (user_id,)) + + conn.commit() # 提交事务 + + cursor.close() + conn.close() + + return result_1, result_2, result_3 + +def clear_table(): + conn = sqlite3.connect('./db/github.db') + cursor = conn.cursor() + cursor.execute('DELETE FROM repos') + conn.commit() + conn.close() + +def record_md(result, filename="./history/github.md"): + # 读取现有内容 + if os.path.exists(filename): + with open(filename, 'r', encoding='utf-8') as file: + existing_content = file.read() + else: + existing_content = "" + + # 将新内容插入到现有内容的开头 + new_content = result + existing_content + + # 写回文件 + with open(filename, 'w', encoding='utf-8') as file: + file.write(new_content) + +def get_filtered_articles(entries, Is_short, choice): + result = "" + record = "" + for entry in entries: + if Is_short == False: + if choice == 1: + result += f"关键词【{entry[6]}】发现新项目：[{entry[1]}]({entry[2]})\n" + result += f"项目描述：{entry[3]}\n" + result += f"上传时间：{entry[4]}\n" + result += f"开发语言：{entry[7]}\t\t作者：{entry[5]}\n" + result += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 2: + result += f"项目：[{entry[1]}]({entry[2]})存在更新！！！\n" + result += f"更新描述：{entry[3]}\n" + result += f"更新时间：{entry[4]}\n" + result += f"提交者：{entry[5]}，[点此查看提交详情]({entry[2]})\n" + result += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 3: + result += f"大佬 {entry[5]} 上传了一个新工具：[{entry[1]}]({entry[2]})\n" + result += f"项目描述：{entry[3]}\n" + result += f"上传时间：{entry[4]}\n" + result += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + elif Is_short == True: + if choice == 1: + result += f"关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n" + result += f"上传时间：{entry[4]}\n" + result += f"开发语言：{entry[6]}\t\t作者：{entry[5]}\n" + result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 2: + result += f"项目：[{entry[1]}]({entry[2]})存在更新！！！\n" + result += f"更新描述：{entry[3]}\n" + result += f"更新时间：{entry[4]}\n" + result += f"提交者：{entry[5]}，[点此查看提交详情]({entry[2]})\n" + result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 3: + result += f"大佬 {entry[5]} 上传了一个新工具：[{entry[1]}]({entry[2]})\n" + result += f"项目描述：{entry[3]}\n" + result += f"上传时间：{entry[4]}\n" + result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章 + + if choice == 1: + record += f"#### 关键词【{entry[7]}】发现新项目：[{entry[1]}]({entry[2]})\n" + record += f"**项目描述**：{entry[3]}\n" + record += f"**上传时间**：{entry[4]}\n" + record += f"**开发语言**：{entry[6]}\t\t**作者**：{entry[5]}\n" + record += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 2: + record += f"#### 项目：[{entry[1]}]({entry[2]})存在更新！！！\n" + record += f"**更新描述**：{entry[3]}\n" + record += f"**更新时间**：{entry[4]}\n" + record += f"**提交者**：{entry[5]}，[点此查看提交详情]({entry[2]})\n" + record += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + elif choice == 3: + record += f"#### 大佬 {entry[5]} 上传了一个新工具：[{entry[1]}]({entry[2]})\n" + record += f"**项目描述**：{entry[3]}\n" + record += f"**上传时间**：{entry[4]}\n" + record += "\n" + "-" * 10 + "\n" # 添加分隔线以便区分不同文章 + + record_md(record) + return result + + +def Src_github(e_hour, Is_short): + if not os.path.exists('./db/github.db'): + # 创建数据库和表 + create_database() + + # 清空表 + clear_table() + + # 插入数据到数据库 + insert_data() + + # 查询指定时间段内的数据 + keyword_data, repo_data, user_data = select_articles(e_hour) + + if keyword_data: + result_1 = get_filtered_articles(keyword_data, Is_short, 1) + else: + result_1 = "" + if repo_data: + result_2 = get_filtered_articles(repo_data, Is_short, 2) + else: + result_2 = "" + if user_data: + result_3 = get_filtered_articles(user_data, Is_short, 3) + else: + result_3 = "" + return result_1, result_2, result_3 + +if __name__ == "__main__": + result_1, result_2, result_3 = Src_github(24, False) + if result_1 != "": + print(result_1) + elif result_2 != "": + print(result_2) + if result_3 != "": + print(result_3) + else: + # 如果为空，则跳过执行 + print("-" * 10) + print("github数据为空，跳过执行。") \ No newline at end of file diff --git a/GotoSend/qianxin.py b/GotoSend/qianxin.py index 62bc2b9..c1b1941 100644 --- a/GotoSend/qianxin.py +++ b/GotoSend/qianxin.py @@ -108,7 +108,7 @@ def get_filtered_articles(entries, Is_short): result += f"上传时间：{entry[5]}\n" result += f"描述：{entry[4]}\n" result += "\n" + "-" * 40 + "\n" # 添加分隔线以便区分不同文章 - if Is_short == False: + if Is_short == True: result += f"文章：[{entry[1]}]({entry[2]})\n" result += f"上传时间：{entry[5]}\n" result += "\n" + "-" * 3 + "\n" # 添加分隔线以便区分不同文章 diff --git a/GotoSend/sougou-wx.py b/GotoSend/sougou_wx.py similarity index 98% rename from GotoSend/sougou-wx.py rename to GotoSend/sougou_wx.py index 24d5966..11ddf2f 100644 --- a/GotoSend/sougou-wx.py +++ b/GotoSend/sougou_wx.py @@ -59,7 +59,7 @@ def get_json(): for keyword, keyword_data in data.items(): # 检查关键词对应的数据是否为列表 if not isinstance(keyword_data, list): - raise ValueError(f"关键词 {keyword} 对应的数据格式错误，请检查common.py是否异常！") + raise ValueError(f"关键词 {keyword} 对应的数据格式错误，请检查爬取程序是否异常！") # 提取所需字段并编号 for index, item in enumerate(keyword_data, start=1): diff --git a/JSON/xianzhi.json b/JSON/xianzhi.json index 21e03eb..867f6d1 100644 --- a/JSON/xianzhi.json +++ b/JSON/xianzhi.json @@ -1,4 +1,154 @@ [ + { + "title": "cjson&json 二进制漏洞利用总结", + "link": "https://xz.aliyun.com/t/16928", + "published": "2024-12-26T22:55:12+08:00", + "id": "https://xz.aliyun.com/t/16928", + "summary": { + "@type": "html", + "#text": "cjson&json 二进制漏洞利用总结" + } + }, + { + "title": "CVE-2024-9047 (WordPress File Upload 插件漏洞分析）", + "link": "https://xz.aliyun.com/t/16927", + "published": "2024-12-26T20:23:12+08:00", + "id": "https://xz.aliyun.com/t/16927", + "summary": { + "@type": "html", + "#text": "CVE-2024-9047 (WordPress File Upload 插件漏洞分析）" + } + }, + { + "title": "Windows应急响应之USBDevices日志", + "link": "https://xz.aliyun.com/t/16926", + "published": "2024-12-26T17:39:57+08:00", + "id": "https://xz.aliyun.com/t/16926", + "summary": { + "@type": "html", + "#text": "Windows应急响应之USBDevices日志" + } + }, + { + "title": "0CTF 2024 部分WP", + "link": "https://xz.aliyun.com/t/16925", + "published": "2024-12-26T17:39:49+08:00", + "id": "https://xz.aliyun.com/t/16925", + "summary": { + "@type": "html", + "#text": "0CTF 2024 部分WP" + } + }, + { + "title": "通过漏洞靶场学习Android应用安全", + "link": "https://xz.aliyun.com/t/16924", + "published": "2024-12-26T17:38:34+08:00", + "id": "https://xz.aliyun.com/t/16924", + "summary": { + "@type": "html", + "#text": "通过漏洞靶场学习Android应用安全" + } + }, + { + "title": "Apache mina CVE-2024-52046漏洞分析复现", + "link": "https://xz.aliyun.com/t/16923", + "published": "2024-12-26T17:30:45+08:00", + "id": "https://xz.aliyun.com/t/16923", + "summary": { + "@type": "html", + "#text": "Apache mina CVE-2024-52046漏洞分析复现" + } + }, + { + "title": "ciscn2024 androidso_re分析", + "link": "https://xz.aliyun.com/t/16920", + "published": "2024-12-26T10:37:32+08:00", + "id": "https://xz.aliyun.com/t/16920", + "summary": { + "@type": "html", + "#text": "ciscn2024 androidso_re分析" + } + }, + { + "title": "前端加解密对抗encrypt-labs", + "link": "https://xz.aliyun.com/t/16919", + "published": "2024-12-26T10:29:25+08:00", + "id": "https://xz.aliyun.com/t/16919", + "summary": { + "@type": "html", + "#text": "前端加解密对抗encrypt-labs" + } + }, + { + "title": "AFL工具使用实践：以libtiff为例的使用指南", + "link": "https://xz.aliyun.com/t/16916", + "published": "2024-12-26T04:16:13+08:00", + "id": "https://xz.aliyun.com/t/16916", + "summary": { + "@type": "html", + "#text": "AFL工具使用实践：以libtiff为例的使用指南" + } + }, + { + "title": "应急响应系列靶场解析", + "link": "https://xz.aliyun.com/t/16914", + "published": "2024-12-25T23:48:42+08:00", + "id": "https://xz.aliyun.com/t/16914", + "summary": { + "@type": "html", + "#text": "应急响应系列靶场解析" + } + }, + { + "title": "记一次某红蓝演练经历", + "link": "https://xz.aliyun.com/t/16913", + "published": "2024-12-25T22:35:21+08:00", + "id": "https://xz.aliyun.com/t/16913", + "summary": { + "@type": "html", + "#text": "记一次某红蓝演练经历" + } + }, + { + "title": "Java安全-WebShell免杀的多种方式", + "link": "https://xz.aliyun.com/t/16912", + "published": "2024-12-25T21:33:55+08:00", + "id": "https://xz.aliyun.com/t/16912", + "summary": { + "@type": "html", + "#text": "Java安全-WebShell免杀的多种方式" + } + }, + { + "title": "Linux渗透实战之Nullbyte靶场提权", + "link": "https://xz.aliyun.com/t/16911", + "published": "2024-12-25T20:46:30+08:00", + "id": "https://xz.aliyun.com/t/16911", + "summary": { + "@type": "html", + "#text": "Linux渗透实战之Nullbyte靶场提权" + } + }, + { + "title": "六安市第二届网络安全大赛WP", + "link": "https://xz.aliyun.com/t/16909", + "published": "2024-12-25T18:21:31+08:00", + "id": "https://xz.aliyun.com/t/16909", + "summary": { + "@type": "html", + "#text": "六安市第二届网络安全大赛WP" + } + }, + { + "title": "PyYAML反序列化学习", + "link": "https://xz.aliyun.com/t/16908", + "published": "2024-12-25T18:12:03+08:00", + "id": "https://xz.aliyun.com/t/16908", + "summary": { + "@type": "html", + "#text": "PyYAML反序列化学习" + } + }, { "title": "JAVA安全之SpEL表达式执行", "link": "https://xz.aliyun.com/t/16905", @@ -848,155 +998,5 @@ "@type": "html", "#text": "第十八届信息安全大赛 && 第二届长城杯 0解PWN题--server解法" } - }, - { - "title": "免杀基础-ppid伪造", - "link": "https://xz.aliyun.com/t/16777", - "published": "2024-12-16T10:00:14+08:00", - "id": "https://xz.aliyun.com/t/16777", - "summary": { - "@type": "html", - "#text": "免杀基础-ppid伪造" - } - }, - { - "title": "高效IO攻击利用学习之House of apple2超详解", - "link": "https://xz.aliyun.com/t/16775", - "published": "2024-12-16T00:37:35+08:00", - "id": "https://xz.aliyun.com/t/16775", - "summary": { - "@type": "html", - "#text": "高效IO攻击利用学习之House of apple2超详解" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - Crypto详解", - "link": "https://xz.aliyun.com/t/16773", - "published": "2024-12-15T22:42:00+08:00", - "id": "https://xz.aliyun.com/t/16773", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - Crypto详解" - } - }, - { - "title": "m0leCon Beginner CTF2024部分wp", - "link": "https://xz.aliyun.com/t/16772", - "published": "2024-12-15T21:31:48+08:00", - "id": "https://xz.aliyun.com/t/16772", - "summary": { - "@type": "html", - "#text": "m0leCon Beginner CTF2024部分wp" - } - }, - { - "title": "Ucum-java XXE漏洞分析(CVE-2024-55887)", - "link": "https://xz.aliyun.com/t/16771", - "published": "2024-12-15T21:04:00+08:00", - "id": "https://xz.aliyun.com/t/16771", - "summary": { - "@type": "html", - "#text": "Ucum-java XXE漏洞分析(CVE-2024-55887)" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - re", - "link": "https://xz.aliyun.com/t/16766", - "published": "2024-12-15T18:21:26+08:00", - "id": "https://xz.aliyun.com/t/16766", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - re" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - zeroshell1~5，WinFT_1~2，sc05_1", - "link": "https://xz.aliyun.com/t/16759", - "published": "2024-12-15T17:26:01+08:00", - "id": "https://xz.aliyun.com/t/16759", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - zeroshell1~5，WinFT_1~2，sc05_1" - } - }, - { - "title": "第二届“长城杯”铁人三项赛 (防护赛)初赛WriteUP", - "link": "https://xz.aliyun.com/t/16789", - "published": "2024-12-15T17:26:01+08:00", - "id": "https://xz.aliyun.com/t/16789", - "summary": { - "@type": "html", - "#text": "第二届“长城杯”铁人三项赛 (防护赛)初赛WriteUP" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - web&re wp", - "link": "https://xz.aliyun.com/t/16755", - "published": "2024-12-15T17:14:30+08:00", - "id": "https://xz.aliyun.com/t/16755", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - web&re wp" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - pwn", - "link": "https://xz.aliyun.com/t/16751", - "published": "2024-12-15T17:05:55+08:00", - "id": "https://xz.aliyun.com/t/16751", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - pwn" - } - }, - { - "title": "第十八届信息安全大赛 && 第二届长城杯 - WEB WP", - "link": "https://xz.aliyun.com/t/16750", - "published": "2024-12-15T17:02:06+08:00", - "id": "https://xz.aliyun.com/t/16750", - "summary": { - "@type": "html", - "#text": "第十八届信息安全大赛 && 第二届长城杯 - WEB WP" - } - }, - { - "title": "HITCTF pipe_snake详解", - "link": "https://xz.aliyun.com/t/16748", - "published": "2024-12-15T09:25:49+08:00", - "id": "https://xz.aliyun.com/t/16748", - "summary": { - "@type": "html", - "#text": "HITCTF pipe_snake详解" - } - }, - { - "title": "记一次客户系统的测试--多种经典类型漏洞", - "link": "https://xz.aliyun.com/t/16747", - "published": "2024-12-14T23:33:38+08:00", - "id": "https://xz.aliyun.com/t/16747", - "summary": { - "@type": "html", - "#text": "记一次客户系统的测试--多种经典类型漏洞" - } - }, - { - "title": "记一次渗透到审计", - "link": "https://xz.aliyun.com/t/16746", - "published": "2024-12-14T21:23:00+08:00", - "id": "https://xz.aliyun.com/t/16746", - "summary": { - "@type": "html", - "#text": "记一次渗透到审计" - } - }, - { - "title": "某次大学的渗透测试经历", - "link": "https://xz.aliyun.com/t/16745", - "published": "2024-12-14T16:51:12+08:00", - "id": "https://xz.aliyun.com/t/16745", - "summary": { - "@type": "html", - "#text": "某次大学的渗透测试经历" - } } ] \ No newline at end of file diff --git a/UpdateLOG.md b/UpdateLOG.md index 032aa21..10be73c 100644 --- a/UpdateLOG.md +++ b/UpdateLOG.md @@ -24,4 +24,6 @@ - 2024年12月18日早：添加了短文本推送机制，一定程度上解决了长字节推送问题（解决办法正在思考中）
- 2024年12月24日晚：上传了测试0.1版本，修复了报错问题
- 2024年12月25日早：优化了代码逻辑和表现
-- 2024年12月25日晚：优化了推送报文格式
\ No newline at end of file +- 2024年12月25日晚：优化了推送报文格式
+- 2024年12月30日：添加并完善了基于搜狗搜索的微信公众号文章监测
+- 2024年12月30日晚：为洞见微信资讯推送添加了关键词筛选
\ No newline at end of file diff --git a/config/__pycache__/check_config.cpython-312.pyc b/config/__pycache__/check_config.cpython-312.pyc index dbefe30521f2d50f488efed4c2d701173cec671d..c525bb6801dfd1709c0c55f3fdd5f249b49239d0 100644 GIT binary patch delta 1091 zcmZvbOK1~87{_<^k$p^eqe+{lt;W{c)?n9IeBcAbXBCBNtyrtIL6c34OxJ8Asc<>R;r`Oi1se!p)&zL|Yh z|F+3^rt2C<$M=UQdqJBtqTt44$8{~OpW}|mV<2u!KAnqwTfKARV4s!Gm-=krWmK&(AB#@7MOF~Jjw|Tmy<2IhXdw>-NLGljDL5Yp zIGS>SpHjg}8j@adF=-Z6v7};^K_En0#Jazbu#ae1^O0)|ff!bhy8j>%AF07LRYb3! zucD-5y~0Hqx9iX98hp;cBIdDxC9Gf#*I*-~^l`0R7w3Q?54Zsi&}RZ%%%2ekxFU!* z5=UAeR13KgD>*{Xz!Rz7yDNPMYlEJc$>l9Ckg`u_a#;^nf>I~#T#7j>D~ixQqbQR` zCQanEyr>o}rM4%JqI|Mwop6G5*GvLRo7P3G1#~)>$y#MgG*ovbG6iQEI^C%GB9tia zoz=q=-RFnyol|09)uySyG`w^4k?Hmhx{O=@hH5^Cz*D7asG{Rw&06)J9AggY3W& m>owXydSRosnVRfBb&ioy*jpP18Sv#0kif#Ph73%^Zpu$M;t$CH delta 742 zcmaKqO=uHA6vuZmJNubUQqweTZKWkmnqY(EQcG3v19K6<+JcEz{K#%%WOZTESc?PgeN6coJ*5h@h#Ui9K6MS9tTC*xg;R+Nfw?WV^J%>TXlzxRF&!+brs6%Tzf3{8O5 z*}PpLYBMy5UN$FRi7O~$G}}ALA4j2fc6M>zvdh^8$FZ|#9Mf97uyWd28q1D#o~(a+ z@rHNMMNNiA410Tup9-}2&%k(6HkDQYELbhH7aN!%QyqfCbD<$J(bV{HJgV#ab+U@% zyoiH@&;XNG$vA(2)ie^8`t|^VV1~NQNB~&ahsjI^*cUu%M7s5c!W572r#In%Pf(fa z6QcVBjcGv0^>>XP7m4{8o#{SipgYwFvLIki#Jffh%%G1k7-j(`GMTANXGU4(*GWt* z-^WE^8D)m}Loy>tr7A7i8z0DDlnuFJxoTUkRC4Z?s}+d1oNCFUF?cr`_rwF!VNXW* zFL@;7%5`dATefc2LSXW=lGKhs5`kTN%r7Y?!$#y;er;y6f1({tel{jI)|EpjJH2-7 zwf^?%`*ZENLR&Ai#lp^cgh=YU0P?%0!}!yIyox`nBp>Cw>bpb=tPG|J2z0`e%ay2g$GK;@_=agx@1``zuNcGHG^F15 WsulPnV=kOVMYO$u225ySXj-8Z3XN1*G^vV0DMcx^Uc^#s4t5DGX=W0)-H$lCOPU@e z2M-Yn9@{^mQV(7PkM-;&L4`84h zL^EL}^&|oTND50>#<8~p2V!*E9nc4P9BqvE+xUlM@Bl?jPh%9!10muFNE$1noah(0 zrv@I!cn=c+mPP%dp3#SK@+Iga4Rjf9f{W_BvlT#m?qs*b6{sUE^(!d4s3DeKCeMB1 zdJ9`+Qu7SQ=CPRRRI7w;kL9Pzj$JXTQ&q!z=C7|sEZ-v_Pt7OnkOj4E<4z; zt1Q{u={cLkW~%2n=9+1^9!qYLM%|$}Vyf>F8oK46k^Tq-3RXAE%`SL7DFD2l2mpO~ z5HvtTi<-dX8Z~T>Dm>*4bh(A|>tv?*gqW3FfuEnd?_g3~YOFh6uE1|vvsXN+NX_yXjs$F7&<1#6{zd|G7INxv`n<=>D5znv) z%k;nGNncc#2G?RETb7w89wj8dU=oYiUV-~e4nr*Qe@s{?gzw-}pGrZlV;( z=qIShkT`~+$ezHB?~rXn_5?}d?tq1KCzGTxMrCa>*3>`YF7xNCGuAp%hNRK zpT0ffKv<@mEWzEkZ{O~|ef!?N_wL_qHVcCCcf+FpK@mcqWLU0JDfe3=u?w zxYY%TkZw>%Y~u8TdUcE(B-Pe1Xi!_@pb=Vq&=jHuDV3HCnnRXB3%~}($e0+4F*6p1 zW~_{ju`>lsAyf2@b&zIhrg&J#l)O$1S{cXdXwb&&fZEQKLS4WXGG!N@Mo6&v5mWvn zVz5Yu*aEib8nMYQfhLFvqP}7scYEsz#detWMkQ`I8tmhv5kM54VPA=|qI7iJ$3`TM z4+G8^6r5S!Lu4A0Q4)fzp>S*ks#Ewf%BT^{X;EhCroX?B5Y#PUyZ;d~W z4Fx-1>1u9lYuMAYM}-c=I$|U4BfSl=k~PR4u??;vK_OZcis60H5F3^hD(oHQ{4k`DVLr^pbO&2v#%Nb_gNsur zN$_$(RuE&B3oILPdxP94TW^h7&WfzyKIF&qjfZ?#h_Qmr*8QS4?2mf=Yz}cW?Bf|O z?2lEPjravG!@5s~Mb;M;SaTrcYbhOg6I%c2Mb{p_936R$-vGcF)lotk;@_LcRwFR$IW(eA3ZD3q{ixuT-? zvyxH>`YE!WVP5cvl2?j~3W<$PG4*e5Ea00`Ok-Xl3>v6dSk2%?fy5&QRuFih-k|8C zLQsJB4+(f{yp0&)K}{8c74aG_Tpl&C?dSX0;IO-&=Y#Gepqb}SPYm&ryC0PHe28T@ z?|G4r`uV75Y`iftp;(5oPJ6JW3;5g>@GrgvRRV43Ooon?lB(Ixna+9r)pNJ}OC`sW zRE8>Crk+SsPb^WpGX-T+@yU3Ka{kW>U9wD9rRl0K5Mek@tWfqT`=ouYY`$dPkgDxY zl^y+0>X=HjGfnSYrkm1q6DIAzl-5b>^w^v@$EJ4gTktOQrFOI}Q|)Q0{eL&~K=Mee z0*+p5-H$$OI9#j0UE_rM=cwwiOaF710bpOwLV(T;|Em4`2UyE&p7qG!95-yC6(Z7zalQ}w_AAv>by3sW8awh_9lF99?i!o^07dOZT;iPQXmdco;G6GdG zP+`S;tT3kQltvdO7<1et8wG=GWGwi007J(qnG&jGDuB=WSYfQ%T+zbVWGV*ZaVkvW zb<~;L^AFDLdSUL=X4wqfnST%NisF`UkVr98lD8r))374_d#vb)Tfbq&9oW5IFUxUh zlWoFFvMrzn2wxRLKno4>&`h}|8KweDGH#a&v|-txH1Fa;5?<{-RA1d2qp__v zM!W)cD>`u2`MQra8uRzC0jgOfqj5qgf)^hJ@~N_0f;D8Zu(GaKDHVDAFpLVG76=pe z?r(1K4DsV02t6$Db_tR=#z`a4BgREM&n4`H`VrL`^oE8Q?{hJy%9Q$+GWC-h^QVsZ6G*^v#^=ML3;Q1tPp#v-iWSukxNAf1TEy-8?1)DvA+8)Dd1#OdyJG2*>KJ0#-ML8io+m z1M*Tx3@H@&Jr8zrLJbTH`!J0`gs@Ul^c;@o7?_MVR&2upAIh8kE4J;=W>58TUwDzhdy>MmT`CZ^PTu*(+8@ zP1%lO%iXDpH|M%aVU8)!=J^#{9!3)IW?8{0HjrPK@d}Jb#bFn%`r`abwG_p&Wgo&> zAlnW9;;V2y3A93&&pGGgi}b!k_l6PKil%xdd**7=y9bgzi`K!!(YyA-#PPd@J07AM z1Dz!AI?88H&YYZME}cr6@6e8nqk6V?rgy$|(eYIB=$-usr%x_B>e7z7tNkfQ-I8N( z>gZX(*VIq%n=Ac`uB+cocHb!|nd`oC>hh_p$CjL}nX;CJ?sVCK8Cbd|c{EeMQ2U=J~pX>N)db z`N5>^PW$oco@K|Lv}4bLBjwn$vUadyWBSaQhL9Cu5GbuBt@0qp{p{^ zhGl1K+S$77Y)d=aZtY%lK6mStwDVB1XQjMqu4l1)Px9o7vu4@pPCMPp&W^ORa=gs$xQa#bv7q^?otIf0LkpA%s9$3RZnJ0cV)_IeoY(8E&q*-W_qIw z1iA4dfvhEoUe#*8V#*}k4c6|5-B96&bo&RC(P_?T+h8L9fz?Ox>-XUAf3kl0Z`a@b z>-!&k|K3mj=E3xvt2aJ+WQa<`?gNUE^M`qXg#gva;+%w4j0}rg;Tf2k_dF;?0vqB- zS&tZn@DS%lqVR1X>BD6BRZgJ0biq`|WXJT_Jhe#IFVoFwx_O~&k#0+oZC^hwj>*;I zen*X&I0W$#>P9cW2VPPK@0bqaYh7Hwm5Io?_#Q_@i3$nNR!abj?y&-Ky1IhqZwPd; zQOZR=axQYmu~RpiGESMMC?DxZKHbYrz=2UVZWrlj84b+9^m-1{gZ-U{GsI00Bbm0c z2RRo}UD91FAtB?6Q*qOb?qUmx;t(0djdALCoB^Uh$ZW{`%tkij+zY}&B<13bz&82K zR&)p_zeXYnzY=N*Y?IZ<1pf2@F<@4vaf^giXOkW~ipPrN&=7%g4ccl0ICg%lFlOy0 z7|V59Q(?&Msx?CR>;a!VuG=*p_MtdLZ~>emJXU0jwwC$<1v0L!^(Np(h^(x~HenLW zJ#HL>*RKUsdJM(~5CMM^T83888$0|>ZanMP4@wgewrg*l=)oSOiwj4hQXR*3ff*gB zgAXUdL|61Q@OgiB9_(!G&u^|y{pISvynpZGAFjS}{oYS!SEp{e`*5&xfBKyV|Ma8P zH{V!&|D$ip^;X~bVEyg)*KWMk?uz1!H`o6>F&*6xh|d!<5B}=)-?@G^g-lMb{^=D+ zA=h!%c5EG63mCkW4TCMS9e|v`%d*XZKiB)NZA4+AI6~(#g22L z1=1aR036G$GsvSLspn0l*1U0^51c!3cW94&b?cWla{1*~|& zF~d%=v^z;;;G)r-c)rc z6^^Dtqp7N~CF^*`>bO_zoIAf*yeDb8JLp}m97@6O%pZM1{+s2ten znQnQ@wowg?e)%KRQ@#gCEj5>V(*6ey>>i$3f+1ico*B1*o8dmf} z;xlTfC=5eaU~n~ptHXG-MEhXU`1rRE{J}FSL^st~{q%vKpG{rHYOG#AW)Lf<$0LL> zt`%1}2-dQb5YI$|>~n$)I2;O#e*zVJRv?I9qJm$c>R%#S{h8Hn^RG|^C`buQWbCIh zW&N45J((wJGNmr{=ggE=W(vzQ7DuLd=dUYlv@T)V*omrZuAX^_D5B#FM MAX_DESCRIPTION_LENGTH: + # 找到300字节处的索引 + byte_index = 0 + char_index = 0 + while byte_index < MAX_DESCRIPTION_LENGTH and char_index < len(description): + byte_index += len(description[char_index].encode('utf-8')) + char_index += 1 + description = description[:char_index - 1] + '...' + + link = result['items'][i]['html_url'] + name = result['items'][i]['name'] + created_at = result['items'][i]['created_at'] + author = result['items'][i]['owner']['login'] + language = result['items'][i]['language'] + + # 将每个项目的详细信息存储在一个字典中 + project_info = { + 'link': link, + 'name': name, + 'created_at': created_at, + 'description': description, + 'author': author, + 'language': language, + 'keyword': keyword + } + # print(project_info) + all_results.append(project_info) + time.sleep(10) + + # 将所有结果写入JSON文件 + save_to_json(all_results, './JSON/github_keyword.json') + + +def github_main_repo(key): + all_results = [] # 用于存储所有结果 + for keyword in key: + logger.info(f"github_repo:项目【{keyword}】更新情况获取开始。") + api_node = "https://api.github.com/repos/{}/commits?per_page=1".format(keyword) + result = fetch_rss(api_node) + + commit = result[0] # 获取最新的提交记录 + description = commit['commit']['message'] + + if description is None: + pass + # 截断描述并在300字节处添加... + elif len(description.encode('utf-8')) > MAX_DESCRIPTION_LENGTH: + byte_index = 0 + char_index = 0 + while byte_index < MAX_DESCRIPTION_LENGTH and char_index < len(description): + byte_index += len(description[char_index].encode('utf-8')) + char_index += 1 + description = description[:char_index - 1] + '...' + + author = commit['commit']['author']['name'] + updated_at = commit['commit']['author']['date'] + link_2 = commit['html_url'] + + # 将每个项目的详细信息存储在一个字典中 + project_info = { + 'link': f"https://api.github.com/{keyword}", + 'name': keyword, + 'updated_at': updated_at, + 'description': description, + 'author': author, + 'link_2': link_2, + 'keyword': keyword + } + # print(project_info) + all_results.append(project_info) + time.sleep(10) + + # 将所有结果写入JSON文件 + save_to_json(all_results, './JSON/github_repo.json') + +def github_main_user(key): + all_results = [] # 用于存储所有结果 + for keyword in key: + logger.info(f"github_user:作者【{keyword}】更新情况获取开始。") + api_node = "https://api.github.com/users/{}/repos?sort=created&per_page=10".format(keyword) + result = fetch_rss(api_node) + for i in range(0, len(result)): + description = result[i]['description'] + + if description is None: + pass + # 检查描述中是否包含黑名单词汇 + elif any(word in description for word in black_words): + continue # 跳过本次执行 + # 截断描述并在300字节处添加... + elif len(description.encode('utf-8')) > MAX_DESCRIPTION_LENGTH: + # 找到300字节处的索引 + byte_index = 0 + char_index = 0 + while byte_index < MAX_DESCRIPTION_LENGTH and char_index < len(description): + byte_index += len(description[char_index].encode('utf-8')) + char_index += 1 + description = description[:char_index - 1] + '...' + + link = result[i]['html_url'] + name = result[i]['name'] + created_at = result[i]['created_at'] + author = result[i]['owner']['login'] + language = result[i]['language'] + + # 将每个项目的详细信息存储在一个字典中 + project_info = { + 'link': link, + 'name': name, + 'created_at': created_at, + 'description': description, + 'author': author, + 'language': language, + 'keyword': keyword + } + # print(project_info) + all_results.append(project_info) + time.sleep(10) + + # 将所有结果写入JSON文件 + save_to_json(all_results, './JSON/github_user.json') + +def github_main(): + if keyword_list: + github_main_keyword(keyword_list) + if tool_list: + github_main_repo(tool_list) + if user_list: + github_main_user(user_list) + +if __name__ == "__main__": + github_main() \ No newline at end of file diff --git a/media/sougou-wx.py b/spider/sougou_wx.py similarity index 95% rename from media/sougou-wx.py rename to spider/sougou_wx.py index b12d7d7..8131ae8 100644 --- a/media/sougou-wx.py +++ b/spider/sougou_wx.py @@ -27,9 +27,10 @@ def fetch_html(url, headers=headers, timeout=10): response = requests.get(url, headers=headers, timeout=timeout) response.raise_for_status() return response.text + except requests.Timeout: + logger.warning(f"请求 {url} 超时，跳过保存操作。") except requests.exceptions.RequestException as e: - print(f"请求出错: {e}") - return None + logger.error(f"请求 {url} 时发生错误: {e}") def parse_html(html_content): soup = BeautifulSoup(html_content, 'html.parser') diff --git a/media/xianzhi.py b/spider/xianzhi.py similarity index 100% rename from media/xianzhi.py rename to spider/xianzhi.py