-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcrawler_java.py
More file actions
113 lines (97 loc) · 4.19 KB
/
Copy pathcrawler_java.py
File metadata and controls
113 lines (97 loc) · 4.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import requests
import json
import os
import subprocess
import time
import concurrent.futures
# 替换为你自己的 GitHub 个人访问 token
TOKEN = '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'
GITHUB_API_URL = 'https://api.github.com/search/repositories'
GITHUB_REPO_API_URL = 'https://api.github.com/repos/'
# 定义请求头，添加身份验证
headers = {
'Authorization': f'token {TOKEN}',
'Accept': 'application/vnd.github.v3+json'
}
# 搜索条件：Java语言，stars大于5000
params = {
'q': 'language:Java stars:>=50000',
'sort': 'stars',
'order': 'desc',
'per_page': 100 # 每页返回100个结果，最多可以获取1000个结果
}
def fetch_projects():
page = 1
all_projects = []
while True:
params['page'] = page
response = requests.get(GITHUB_API_URL, headers=headers, params=params)
if response.status_code == 200:
data = response.json()
items = data.get('items', [])
if not items:
break # 如果没有更多项目了，就停止爬取
all_projects.extend(items)
print(f'Page {page}: Fetched {len(items)} projects')
page += 1
else:
print(f'Error: {response.status_code}')
break
return all_projects
def save_to_json(projects):
with open('high_quality_java_projects.json', 'w', encoding='utf-8') as f:
json.dump(projects, f, ensure_ascii=False, indent=4)
print('Saved projects to high_quality_java_projects.json')
def get_default_branch(repo_full_name):
"""获取项目的默认分支"""
try:
url = f'{GITHUB_REPO_API_URL}{repo_full_name}'
response = requests.get(url, headers=headers)
if response.status_code == 200:
repo_data = response.json()
return repo_data.get('default_branch', 'main') # 默认返回main
else:
print(f"Error fetching repo info for {repo_full_name}: {response.status_code}")
return 'main' # 如果无法获取默认分支，默认使用main
except Exception as e:
print(f"Exception while fetching default branch for {repo_full_name}: {e}")
return 'main'
def clone_project(repo_url, repo_full_name, clone_dir, retries=3, delay=5):
"""克隆单个项目，并处理重试逻辑"""
try:
# 获取默认分支
default_branch = get_default_branch(repo_full_name)
print(f'Cloning from {repo_url} ({default_branch} branch) to {clone_dir}')
subprocess.run(['git', 'clone', '--single-branch', '--branch', default_branch, repo_url, clone_dir], check=True)
print(f'Successfully cloned {repo_url}')
except subprocess.CalledProcessError as e:
if retries > 0:
print(f'Error cloning {repo_url}, retrying... {retries} attempts left')
time.sleep(delay) # 等待一段时间后重试
clone_project(repo_url, repo_full_name, clone_dir, retries - 1, delay)
else:
print(f'Failed to clone {repo_url} after multiple attempts: {e}')
def clone_projects(projects, base_dir='../dataset/train_data/java_projects'):
if not os.path.exists(base_dir):
os.makedirs(base_dir)
# 使用 ThreadPoolExecutor 进行并行克隆
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
futures = []
for project in projects:
repo_url = project['html_url']
repo_full_name = project['full_name'] # 获取完整的仓库名称（例如：username/repository）
repo_name = project['name']
clone_dir = os.path.join(base_dir, repo_name)
if not os.path.exists(clone_dir):
futures.append(executor.submit(clone_project, repo_url, repo_full_name, clone_dir))
else:
print(f'{repo_name} already cloned, skipping.')
# 等待所有克隆任务完成
concurrent.futures.wait(futures)
print('All cloning tasks completed.')
if __name__ == '__main__':
print('Fetching projects...')
projects = fetch_projects()
save_to_json(projects)
print('Cloning projects...')
clone_projects(projects)