PyBot/github开发文档.md

### 设计思路
前情提要：GitHub的API接口为json格式，极其方便于使用python进行提取分析
api地址：
获取关键词下更新的最新仓库源：https://api.github.com/search/repositories?q={Keyword}&sort=updated&per_page=30
- sort=updated：按更新时间排序
- per_page=30：查询数量，建议设置为30
- page=1：指定查询页数
获取指定用户的仓库更新情况：https://api.github.com/users/{user}/repos
获取指定仓库commit情况：https://api.github.com/repos/{user}/{repo}

### 速率限制
headers ={ "Authorization": " token OAUTH-TOKEN"}
OAUTH-TOKEN：github个人账号设置->开发者设置->个人token。创建一个新token时，可以选择具体的权限，创建成功时一定要复制到本地哪里保存，只会让你看见一次，如果忘记的话就需要重新生成。

### 使用技术
- python-json解析
- python-sqlite联动
- python-request爬虫
- sqlite筛选

### 参考Json源格式
所需部分：
- html_url
- created_at：仓库创建时间
- updated_at：仓库最近更新时间
- pushed_at：仓库最近推送时间（参考此元素进行设计）
- description：仓库描述

{
  "id": 511095846,
  "node_id": "R_kgDOHna0Jg",
  "name": "TestnetProtocol",
  "full_name": "exorde-labs/TestnetProtocol",
  "private": false,
  "owner": {
    "login": "exorde-labs",
    "id": 64810085,
    "node_id": "MDEyOk9yZ2FuaXphdGlvbjY0ODEwMDg1",
    "avatar_url": "https://avatars.githubusercontent.com/u/64810085?v=4",
    "gravatar_id": "",
    "url": "https://api.github.com/users/exorde-labs",
    "html_url": "https://github.com/exorde-labs",
    "followers_url": "https://api.github.com/users/exorde-labs/followers",
    "following_url": "https://api.github.com/users/exorde-labs/following{/other_user}",
    "gists_url": "https://api.github.com/users/exorde-labs/gists{/gist_id}",
    "starred_url": "https://api.github.com/users/exorde-labs/starred{/owner}{/repo}",
    "subscriptions_url": "https://api.github.com/users/exorde-labs/subscriptions",
    "organizations_url": "https://api.github.com/users/exorde-labs/orgs",
    "repos_url": "https://api.github.com/users/exorde-labs/repos",
    "events_url": "https://api.github.com/users/exorde-labs/events{/privacy}",
    "received_events_url": "https://api.github.com/users/exorde-labs/received_events",
    "type": "Organization",
    "user_view_type": "public",
    "site_admin": false
  },
  "html_url": "https://github.com/exorde-labs/TestnetProtocol",
  "description": null,
  "fork": false,
  "url": "https://api.github.com/repos/exorde-labs/TestnetProtocol",
  "forks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/forks",
  "keys_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/keys{/key_id}",
  "collaborators_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/collaborators{/collaborator}",
  "teams_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/teams",
  "hooks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/hooks",
  "issue_events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/events{/number}",
  "events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/events",
  "assignees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/assignees{/user}",
  "branches_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/branches{/branch}",
  "tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/tags",
  "blobs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/blobs{/sha}",
  "git_tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/tags{/sha}",
  "git_refs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/refs{/sha}",
  "trees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/trees{/sha}",
  "statuses_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/statuses/{sha}",
  "languages_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/languages",
  "stargazers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/stargazers",
  "contributors_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contributors",
  "subscribers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscribers",
  "subscription_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscription",
  "commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/commits{/sha}",
  "git_commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/commits{/sha}",
  "comments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/comments{/number}",
  "issue_comment_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/comments{/number}",
  "contents_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contents/{+path}",
  "compare_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/compare/{base}...{head}",
  "merges_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/merges",
  "archive_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/{archive_format}{/ref}",
  "downloads_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/downloads",
  "issues_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues{/number}",
  "pulls_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/pulls{/number}",
  "milestones_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/milestones{/number}",
  "notifications_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/notifications{?since,all,participating}",
  "labels_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/labels{/name}",
  "releases_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/releases{/id}",
  "deployments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/deployments",
  "created_at": "2022-07-06T10:44:29Z",
  "updated_at": "2024-12-27T02:20:32Z",
  "pushed_at": "2024-12-27T02:20:28Z",
  "git_url": "git://github.com/exorde-labs/TestnetProtocol.git",
  "ssh_url": "git@github.com:exorde-labs/TestnetProtocol.git",
  "clone_url": "https://github.com/exorde-labs/TestnetProtocol.git",
  "svn_url": "https://github.com/exorde-labs/TestnetProtocol",
  "homepage": null,
  "size": 1918317,
  "stargazers_count": 16,
  "watchers_count": 16,
  "language": "Solidity",
  "has_issues": true,
  "has_projects": true,
  "has_downloads": true,
  "has_wiki": true,
  "has_pages": false,
  "has_discussions": false,
  "forks_count": 20,
  "mirror_url": null,
  "archived": false,
  "disabled": false,
  "open_issues_count": 0,
  "license": {
    "key": "mit",
    "name": "MIT License",
    "spdx_id": "MIT",
    "url": "https://api.github.com/licenses/mit",
    "node_id": "MDc6TGljZW5zZTEz"
  },
  "allow_forking": true,
  "is_template": false,
  "web_commit_signoff_required": false,
  "topics": [

  ],
  "visibility": "public",
  "forks": 20,
  "open_issues": 0,
  "watchers": 16,
  "default_branch": "main",
  "score": 1.0
}

### 参考代码
year = datetime.datetime.now().year
api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year)
json_str = requests.get(api, headers=github_headers, timeout=10).json()