Skip to content

Commit 8960716

Browse files
null-nicknull-nick
committed
improved scrape errors script
1 parent da0a6d6 commit 8960716

1 file changed

Lines changed: 68 additions & 40 deletions

File tree

‎compiler/errors/sort.py‎

Lines changed: 68 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -42,45 +42,73 @@
4242
f.write("\n")
4343

4444
elif sys.argv[1] == "scrape":
45-
b = "https://core.telegram.org"
46-
c = "/api/errors"
47-
a = requests.get(b + c)
48-
d = a.text
49-
e = r"\<a\ href\=\"(.*)\"\>here.*\<\/a\>"
50-
f = re.search(e, d)
51-
if f:
52-
a = requests.get(
53-
b + f.group(1)
54-
)
55-
d = a.json()
56-
e = d.get("errors", [])
57-
for h in e:
58-
dct = {}
59-
60-
j = d.get("errors").get(h)
61-
for k in j:
62-
if k.endswith("_*"):
63-
continue
64-
g = d.get("descriptions")
65-
l = g.get(k)
66-
m = k.replace("_%d", "_X")
67-
l = l.replace("%d", "{value}")
68-
l = l.replace("&raquo;", "»")
69-
l = l.replace("&laquo;", "«")
70-
l = l.replace("](/api/", f"]({b}/api/")
71-
dct[m] = l
72-
73-
for p in Path("source/").glob(f"{h}*.tsv"):
74-
with open(p) as f:
75-
reader = csv.reader(f, delimiter="\t")
76-
for k, v in reader:
77-
if k != "id":
78-
dct[k] = v
45+
base_url = "https://core.telegram.org"
46+
errors_api_path = "/api/errors"
47+
errors_data = None
48+
html_content = None
7949

80-
keys = sorted(dct)
50+
try:
51+
response = requests.get(base_url + errors_api_path)
52+
response.raise_for_status()
53+
html_content = response.text
54+
except requests.exceptions.RequestException as e:
55+
print(f"Error fetching errors page: {e}")
56+
57+
match = re.search(r'<a href=\"(.*?)\">here.*?</a>', html_content)
58+
if not match:
59+
print("Link to errors JSON not found.")
60+
61+
errors_json_url = base_url + match.group(1)
62+
try:
63+
response = requests.get(errors_json_url)
64+
response.raise_for_status()
65+
errors_data = response.json()
66+
except requests.exceptions.RequestException as e:
67+
print(f"Error fetching errors JSON: {e}")
68+
except ValueError:
69+
print("Error decoding JSON response.")
70+
71+
error_categories = errors_data.get("errors", [])
72+
if not error_categories:
73+
print("No error categories found in JSON.")
74+
75+
source_dir = Path("source/")
76+
source_dir.mkdir(exist_ok=True)
77+
78+
for error_type in error_categories:
79+
data_dict = {}
80+
81+
for tsv_path in source_dir.glob(f"{error_type}*.tsv"):
82+
with open(tsv_path, 'r') as tsv_file:
83+
reader = csv.DictReader(tsv_file, delimiter='\t')
84+
for row in reader:
85+
if row['id'] != 'id':
86+
data_dict[row['id']] = row['message']
87+
88+
error_details = errors_data["errors"].get(error_type, {})
89+
descriptions = errors_data["descriptions"]
90+
91+
for error_code in error_details:
92+
if error_code.endswith("_*"):
93+
continue
94+
95+
description = descriptions.get(error_code, "")
96+
if description:
97+
processed_description = (
98+
description.replace("%d", "{value}")
99+
.replace("\"", "'")
100+
.replace("»", "»")
101+
.replace("«", "«")
102+
.replace(" »", "")
103+
.replace("](/api/", f"]({base_url}/api/")
104+
)
105+
data_dict[error_code.replace("_%d", "_X")] = processed_description
106+
107+
sorted_keys = sorted(data_dict.keys())
81108

82-
for p in Path("source/").glob(f"{h}*.tsv"):
83-
with open(p, "w") as f:
84-
f.write("id\tmessage\n")
85-
for i, item in enumerate(keys, start=1):
86-
f.write(f"{item}\t{dct[item]}\n")
109+
for tsv_path in source_dir.glob(f"{error_type}*.tsv"):
110+
print(f"Writing to {tsv_path}")
111+
with open(tsv_path, 'w', newline='') as tsv_file:
112+
writer = csv.writer(tsv_file, delimiter='\t')
113+
writer.writerow(["id", "message"])
114+
writer.writerows([(key, data_dict[key]) for key in sorted_keys])

0 commit comments

Comments
 (0)