143 lines
7.2 KiB
Markdown
143 lines
7.2 KiB
Markdown
### 设计思路
|
||
前情提要：GitHub的API接口为json格式，极其方便于使用python进行提取分析
|
||
api地址：
|
||
获取关键词下更新的最新仓库源：https://api.github.com/search/repositories?q={Keyword}&sort=updated&per_page=30
|
||
- sort=updated：按更新时间排序
|
||
- per_page=30：查询数量，建议设置为30
|
||
- page=1：指定查询页数
|
||
获取指定用户的仓库更新情况：https://api.github.com/users/{user}/repos
|
||
获取指定仓库commit情况：https://api.github.com/repos/{user}/{repo}
|
||
|
||
### 速率限制
|
||
headers ={ "Authorization": " token OAUTH-TOKEN"}
|
||
OAUTH-TOKEN：github个人账号设置->开发者设置->个人token。创建一个新token时，可以选择具体的权限，创建成功时一定要复制到本地哪里保存，只会让你看见一次，如果忘记的话就需要重新生成。
|
||
|
||
### 使用技术
|
||
- python-json解析
|
||
- python-sqlite联动
|
||
- python-request爬虫
|
||
- sqlite筛选
|
||
|
||
### 参考Json源格式
|
||
所需部分：
|
||
- html_url
|
||
- created_at：仓库创建时间
|
||
- updated_at：仓库最近更新时间
|
||
- pushed_at：仓库最近推送时间（参考此元素进行设计）
|
||
- description：仓库描述
|
||
|
||
{
|
||
"id": 511095846,
|
||
"node_id": "R_kgDOHna0Jg",
|
||
"name": "TestnetProtocol",
|
||
"full_name": "exorde-labs/TestnetProtocol",
|
||
"private": false,
|
||
"owner": {
|
||
"login": "exorde-labs",
|
||
"id": 64810085,
|
||
"node_id": "MDEyOk9yZ2FuaXphdGlvbjY0ODEwMDg1",
|
||
"avatar_url": "https://avatars.githubusercontent.com/u/64810085?v=4",
|
||
"gravatar_id": "",
|
||
"url": "https://api.github.com/users/exorde-labs",
|
||
"html_url": "https://github.com/exorde-labs",
|
||
"followers_url": "https://api.github.com/users/exorde-labs/followers",
|
||
"following_url": "https://api.github.com/users/exorde-labs/following{/other_user}",
|
||
"gists_url": "https://api.github.com/users/exorde-labs/gists{/gist_id}",
|
||
"starred_url": "https://api.github.com/users/exorde-labs/starred{/owner}{/repo}",
|
||
"subscriptions_url": "https://api.github.com/users/exorde-labs/subscriptions",
|
||
"organizations_url": "https://api.github.com/users/exorde-labs/orgs",
|
||
"repos_url": "https://api.github.com/users/exorde-labs/repos",
|
||
"events_url": "https://api.github.com/users/exorde-labs/events{/privacy}",
|
||
"received_events_url": "https://api.github.com/users/exorde-labs/received_events",
|
||
"type": "Organization",
|
||
"user_view_type": "public",
|
||
"site_admin": false
|
||
},
|
||
"html_url": "https://github.com/exorde-labs/TestnetProtocol",
|
||
"description": null,
|
||
"fork": false,
|
||
"url": "https://api.github.com/repos/exorde-labs/TestnetProtocol",
|
||
"forks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/forks",
|
||
"keys_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/keys{/key_id}",
|
||
"collaborators_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/collaborators{/collaborator}",
|
||
"teams_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/teams",
|
||
"hooks_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/hooks",
|
||
"issue_events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/events{/number}",
|
||
"events_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/events",
|
||
"assignees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/assignees{/user}",
|
||
"branches_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/branches{/branch}",
|
||
"tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/tags",
|
||
"blobs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/blobs{/sha}",
|
||
"git_tags_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/tags{/sha}",
|
||
"git_refs_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/refs{/sha}",
|
||
"trees_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/trees{/sha}",
|
||
"statuses_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/statuses/{sha}",
|
||
"languages_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/languages",
|
||
"stargazers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/stargazers",
|
||
"contributors_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contributors",
|
||
"subscribers_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscribers",
|
||
"subscription_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/subscription",
|
||
"commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/commits{/sha}",
|
||
"git_commits_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/git/commits{/sha}",
|
||
"comments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/comments{/number}",
|
||
"issue_comment_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues/comments{/number}",
|
||
"contents_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/contents/{+path}",
|
||
"compare_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/compare/{base}...{head}",
|
||
"merges_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/merges",
|
||
"archive_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/{archive_format}{/ref}",
|
||
"downloads_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/downloads",
|
||
"issues_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/issues{/number}",
|
||
"pulls_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/pulls{/number}",
|
||
"milestones_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/milestones{/number}",
|
||
"notifications_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/notifications{?since,all,participating}",
|
||
"labels_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/labels{/name}",
|
||
"releases_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/releases{/id}",
|
||
"deployments_url": "https://api.github.com/repos/exorde-labs/TestnetProtocol/deployments",
|
||
"created_at": "2022-07-06T10:44:29Z",
|
||
"updated_at": "2024-12-27T02:20:32Z",
|
||
"pushed_at": "2024-12-27T02:20:28Z",
|
||
"git_url": "git://github.com/exorde-labs/TestnetProtocol.git",
|
||
"ssh_url": "git@github.com:exorde-labs/TestnetProtocol.git",
|
||
"clone_url": "https://github.com/exorde-labs/TestnetProtocol.git",
|
||
"svn_url": "https://github.com/exorde-labs/TestnetProtocol",
|
||
"homepage": null,
|
||
"size": 1918317,
|
||
"stargazers_count": 16,
|
||
"watchers_count": 16,
|
||
"language": "Solidity",
|
||
"has_issues": true,
|
||
"has_projects": true,
|
||
"has_downloads": true,
|
||
"has_wiki": true,
|
||
"has_pages": false,
|
||
"has_discussions": false,
|
||
"forks_count": 20,
|
||
"mirror_url": null,
|
||
"archived": false,
|
||
"disabled": false,
|
||
"open_issues_count": 0,
|
||
"license": {
|
||
"key": "mit",
|
||
"name": "MIT License",
|
||
"spdx_id": "MIT",
|
||
"url": "https://api.github.com/licenses/mit",
|
||
"node_id": "MDc6TGljZW5zZTEz"
|
||
},
|
||
"allow_forking": true,
|
||
"is_template": false,
|
||
"web_commit_signoff_required": false,
|
||
"topics": [
|
||
|
||
],
|
||
"visibility": "public",
|
||
"forks": 20,
|
||
"open_issues": 0,
|
||
"watchers": 16,
|
||
"default_branch": "main",
|
||
"score": 1.0
|
||
}
|
||
|
||
### 参考代码
|
||
year = datetime.datetime.now().year
|
||
api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format(year)
|
||
json_str = requests.get(api, headers=github_headers, timeout=10).json() |