diff --git a/spider/github.py b/spider/github.py index 648bf13..385aaa1 100644 --- a/spider/github.py +++ b/spider/github.py @@ -33,8 +33,10 @@ def fetch_rss(url, timeout=10): return response.json() except requests.Timeout: logger.warning(f"请求 {url} 超时,跳过保存操作。") + return None except requests.exceptions.RequestException as e: logger.error(f"请求 {url} 时发生错误: {e}") + return None def save_to_json(data, filename): with open(filename, 'w', encoding='utf-8') as f: @@ -46,6 +48,9 @@ def github_main_keyword(key): logger.info(f"github_keyword:关键词【{keyword}】获取开始。") api_node = "https://api.github.com/search/repositories?q={}&sort=updated&per_page=20".format(keyword) result = fetch_rss(api_node) + if result == None: + time.sleep(5) + continue for i in range(0, 20): description = result['items'][i]['description'] @@ -94,6 +99,9 @@ def github_main_repo(key): logger.info(f"github_repo:项目【{keyword}】更新情况获取开始。") api_node = "https://api.github.com/repos/{}/commits?per_page=1".format(keyword) result = fetch_rss(api_node) + if result == None: + time.sleep(5) + continue commit = result[0] # 获取最新的提交记录 description = commit['commit']['message'] @@ -136,6 +144,9 @@ def github_main_release(key): logger.info(f"github_repo:项目【{keyword}】发版情况获取开始。") api_node = "https://api.github.com/repos/{}/releases?per_page=1".format(keyword) result = fetch_rss(api_node) + if result == None: + time.sleep(5) + continue if not result: logger.warning(f"github_repo:项目【{keyword}】不存在版本发布情况。") time.sleep(5) @@ -168,6 +179,9 @@ def github_main_user(key): logger.info(f"github_user:作者【{keyword}】更新情况获取开始。") api_node = "https://api.github.com/users/{}/repos?sort=created&per_page=10".format(keyword) result = fetch_rss(api_node) + if result == None: + time.sleep(5) + continue for i in range(0, len(result)): description = result[i]['description']