PyBot/github开发文档.md

设计思路

前情提要：GitHub的API接口为json格式，极其方便于使用python进行提取分析 api地址： 获取关键词下更新的最新仓库源：https://api.github.com/search/repositories?q={Keyword}&sort=updated&per_page=30

• sort=updated：按更新时间排序
• per_page=30：查询数量，建议设置为30
• page=1：指定查询页数 获取指定用户的仓库更新情况：https://api.github.com/users/{user}/repos 获取指定仓库commit情况：https://api.github.com/repos/{user}/{repo}

速率限制

headers ={ "Authorization": " token OAUTH-TOKEN"} OAUTH-TOKEN：github个人账号设置->开发者设置->个人token。创建一个新token时，可以选择具体的权限，创建成功时一定要复制到本地哪里保存，只会让你看见一次，如果忘记的话就需要重新生成。

使用技术

• python-json解析
• python-sqlite联动
• python-request爬虫
• sqlite筛选

参考Json源格式

所需部分：

• html_url
• created_at：仓库创建时间
• updated_at：仓库最近更新时间
• pushed_at：仓库最近推送时间（参考此元素进行设计）
• description：仓库描述

{ "id": 511095846, "node_id": "R_kgDOHna0Jg", "name": "TestnetProtocol", "full_name": "exorde-labs/TestnetProtocol", "private": false, "owner": { "login": "exorde-labs", "id": 64810085, "node_id": "MDEyOk9yZ2FuaXphdGlvbjY0ODEwMDg1", "avatar_url": "https://avatars.githubusercontent.com/u/64810085?v=4", "gravatar_id": "", "url": "https://api.github.com/users/exorde-labs", "html_url": "https://github.com/exorde-labs", "followers_url": "https://api.github.com/users/exorde-labs/followers", "following_url": "https://api.github.com/users/exorde-labs/following{/other_user}", "gists_url": "https://api.github.com/users/exorde-labs/gists{/gist_id}", "starred_url": "https://api.github.com/users/exorde-labs/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/exorde-labs/subscriptions", "organizations_url": "https://api.github.com/users/exorde-labs/orgs", "repos_url": "https://api.github.com/users/exorde-labs/repos", "events_url": "https://api.github.com/users/exorde-labs/events{/privacy}", "received_events_url": "https://api.github.com/users/exorde-labs/received_events", "type": "Organization", "user_view_type": "public", "site_admin": false }, "html_url": "https://github.com/exorde-labs/TestnetProtocol", "description": null, "fork": false, "url": "https://api.github.com/repos/exorde-labs/TestnetProtocol", "forks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/forks", "keys_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/teams", "hooks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/hooks", "issue_events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/events{/number}", "events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/events", "assignees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/assignees{/user}", "branches_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/branches{/branch}", "tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/tags", "blobs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/refs{/sha}", "trees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/statuses/{sha}", "languages_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/languages", "stargazers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/stargazers", "contributors_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contributors", "subscribers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscribers", "subscription_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscription", "commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/commits{/sha}", "git_commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/commits{/sha}", "comments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/comments{/number}", "issue_comment_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/comments{/number}", "contents_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contents/{+path}", "compare_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/merges", "archive_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/downloads", "issues_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues{/number}", "pulls_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/pulls{/number}", "milestones_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/milestones{/number}", "notifications_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/labels{/name}", "releases_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/releases{/id}", "deployments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/deployments", "created_at": "2022-07-06T10:44:29Z", "updated_at": "2024-12-27T02:20:32Z", "pushed_at": "2024-12-27T02:20:28Z", "git_url": "git://github.com/exorde-labs/TestnetProtocol.git", "ssh_url": "git@github.com:exorde-labs/TestnetProtocol.git", "clone_url": "https://github.com/exorde-labs/TestnetProtocol.git", "svn_url": "https://github.com/exorde-labs/TestnetProtocol", "homepage": null, "size": 1918317, "stargazers_count": 16, "watchers_count": 16, "language": "Solidity", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 20, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": { "key": "mit", "name": "MIT License", "spdx_id": "MIT", "url": "https://api.github.com/licenses/mit", "node_id": "MDc6TGljZW5zZTEz" }, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [

], "visibility": "public", "forks": 20, "open_issues": 0, "watchers": 16, "default_branch": "main", "score": 1.0 }

参考代码

year = datetime.datetime.now().year api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year) json_str = requests.get(api, headers=github_headers, timeout=10).json()