-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathextract_software.py
More file actions
150 lines (113 loc) · 4.36 KB
/
Copy pathextract_software.py
File metadata and controls
150 lines (113 loc) · 4.36 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
#!/usr/bin/env python3
"""Extract software data from existing HTML into Excel spreadsheet.
This is a one-time script to migrate existing HTML data to the spreadsheet format.
"""
import re
from pathlib import Path
from bs4 import BeautifulSoup
import openpyxl
def extract_software(html_path: Path) -> dict:
"""Extract all software data from HTML file.
Returns dict with keys: python, javascript, matlab
"""
with open(html_path, 'r', encoding='utf-8') as f:
soup = BeautifulSoup(f.read(), 'html.parser')
sections = {
'python': 'python',
'javascript': 'javascript',
'matlab': 'matlab'
}
data = {}
for key, section_id in sections.items():
section = soup.find('section', id=section_id)
if not section:
print(f"Warning: Section '{section_id}' not found")
data[key] = []
continue
software_list = section.find('div', class_='software-list')
if not software_list:
print(f"Warning: No software-list in '{section_id}'")
data[key] = []
continue
items = []
for p in software_list.find_all('p', recursive=False):
item = extract_software_item(p)
if item['name']:
items.append(item)
data[key] = items
print(f"Extracted {len(items)} items from '{key}'")
return data
def extract_software_item(p) -> dict:
"""Extract data from a single software paragraph.
Format: <strong>Name.</strong> Description. [<a>Link</a>]
"""
item = {
'name': '',
'description': '',
'links_html': ''
}
# Get the name from <strong> tag
strong = p.find('strong')
if strong:
name = strong.get_text(strip=True)
# Remove trailing period if present
item['name'] = name.rstrip('.')
# Get the full inner HTML
inner_html = get_inner_html(p)
# Extract links at the end (text in square brackets)
# Links are usually at the end like [<a href="...">GitHub</a>]
link_match = re.search(r'\[([^\]]*<a[^>]*>[^<]*</a>[^\]]*)\]', inner_html)
if link_match:
item['links_html'] = f'[{link_match.group(1)}]'
# Remove the link from the HTML to get description
inner_html = inner_html[:link_match.start()].strip()
# Now extract description (everything after the name)
# Remove the <strong>...</strong> part
desc_html = re.sub(r'<strong>[^<]*</strong>\s*', '', inner_html)
# Clean up the description
desc_html = desc_html.strip()
# Remove leading period if present
if desc_html.startswith('.'):
desc_html = desc_html[1:].strip()
item['description'] = desc_html
return item
def get_inner_html(element) -> str:
"""Get the inner HTML of an element as a string."""
return ''.join(str(child) for child in element.children).strip()
def save_to_excel(data: dict, output_path: Path):
"""Save extracted data to Excel spreadsheet with multiple sheets."""
wb = openpyxl.Workbook()
wb.remove(wb.active)
sheet_configs = [
('python', ['name', 'description', 'links_html']),
('javascript', ['name', 'description', 'links_html']),
('matlab', ['name', 'description', 'links_html']),
]
for sheet_name, columns in sheet_configs:
ws = wb.create_sheet(title=sheet_name)
# Write headers
for col, header in enumerate(columns, 1):
ws.cell(row=1, column=col, value=header)
# Write data
items = data.get(sheet_name, [])
for row_num, item in enumerate(items, 2):
for col, header in enumerate(columns, 1):
value = item.get(header, '')
ws.cell(row=row_num, column=col, value=value)
# Adjust column widths
for col in range(1, len(columns) + 1):
ws.column_dimensions[openpyxl.utils.get_column_letter(col)].width = 60
wb.save(output_path)
print(f"Saved to {output_path}")
def main():
project_root = Path(__file__).parent.parent
html_path = project_root / 'software.html'
output_path = project_root / 'data' / 'software.xlsx'
print(f"Extracting from: {html_path}")
data = extract_software(html_path)
total = sum(len(items) for items in data.values())
print(f"\nTotal items extracted: {total}")
save_to_excel(data, output_path)
print("Done!")
if __name__ == '__main__':
main()