Repository navigation
Expand file tree
/
Copy pathupdate_ambiguous_characters.py
More file actions
130 lines (104 loc) · 4.27 KB
/
Copy pathupdate_ambiguous_characters.py
File metadata and controls
130 lines (104 loc) · 4.27 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# /// script
# requires-python = ">=3.13"
# dependencies = []
#
# [tool.ty.rules]
# truthiness-test-of-none-union = "warn"
# blanket-ignore-comment = "warn"
# missing-type-argument = "warn"
# possibly-unresolved-reference = "warn"
# unsound-return-statement = "warn"
# unsound-yield = "warn"
# unsupported-dynamic-base = "warn"
# division-by-zero = "warn"
# dynamic-function-decorator-return = "warn"
# unsound-assignment = "warn"
# redundant-condition-strict = "warn"
# disjoint-cast = "warn"
# missing-direct-dependency = "warn"
#
# [tool.uv]
# no-build = true
# exclude-newer = "P7D"
# ///
"""Generate the confusables.rs file from the VS Code ambiguous.json file."""
from __future__ import annotations
import itertools
import json
import subprocess
from pathlib import Path
CONFUSABLES_RS_PATH = "crates/ruff_linter/src/rules/ruff/rules/confusables.rs"
AMBIGUOUS_JSON_URL = "https://raw.githubusercontent.com/hediet/vscode-unicode-data/main/out/ambiguous.json"
prelude = """
//! This file is auto-generated by `scripts/update_ambiguous_characters.py`.
/// Via: <https://github.com/hediet/vscode-unicode-data/blob/main/out/ambiguous.json>
/// See: <https://github.com/microsoft/vscode/blob/095ddabc52b82498ee7f718a34f9dd11d59099a8/src/vs/base/common/strings.ts#L1094>
pub(crate) fn confusable(c: u32) -> Option<char> {
let result = match c {
""".lstrip()
postlude = """_ => return None, }; Some(result)}"""
def get_mapping_data() -> dict[str, list[int]]:
"""
Get the ambiguous character mapping data from the vscode-unicode-data repository.
Uses the system's `curl` command to download the data,
instead of adding a dependency to a Python-native HTTP client.
"""
content = subprocess.check_output(
["curl", "-sSL", AMBIGUOUS_JSON_URL],
encoding="utf-8",
)
# The content is a JSON object literal wrapped in a JSON string, so double decode:
mapping_data = json.loads(json.loads(content))
assert isinstance(mapping_data, dict)
return mapping_data # ty: ignore[unsound-return-statement]
def format_number(number: int) -> str:
"""Underscore-separate the digits of a number."""
# For unknown historical reasons, numbers greater than 100,000 were
# underscore-delimited in the generated file, so we now preserve that property to
# avoid unnecessary churn.
if number > 100000:
number: str = str(number)
number = "_".join(number[i : i + 3] for i in range(0, len(number), 3))
return f"{number}_u32"
return f"{number}u32"
def format_char(number: int) -> str:
"""Format a Python integer as a Rust character literal."""
char = chr(number)
if char == "\\":
return "\\\\"
return char
def format_confusables_rs(raw_data: dict[str, list[int]]) -> str:
"""Format the downloaded data into a Rust source file."""
# The input data contains duplicate entries.
flattened_items: set[tuple[int, int]] = set()
for items in raw_data.values():
assert len(items) % 2 == 0, "Expected pairs of items"
flattened_items.update(itertools.batched(items, 2, strict=True))
tuples = [
f" {format_number(left)} => '{format_char(right)}',\n"
for left, right in sorted(flattened_items)
]
# Add some additional confusable pairs that are not included in the VS Code data,
# as they're unicode-to-unicode confusables, not unicode-to-ASCII confusables.
confusable_units = [
# ANGSTROM SIGN → LATIN CAPITAL LETTER A WITH RING ABOVE
("0x212B", chr(0x00C5)),
# OHM SIGN → GREEK CAPITAL LETTER OMEGA
("0x2126", chr(0x03A9)),
# MICRO SIGN → GREEK SMALL LETTER MU
("0x00B5", chr(0x03BC)),
]
tuples += [f" {left} => '{right}',\n" for left, right in confusable_units]
print(f"{len(tuples)} confusable tuples.")
return prelude + "".join(tuples) + postlude
def main() -> None:
print("Retrieving data...")
mapping_data = get_mapping_data()
formatted_data = format_confusables_rs(mapping_data)
confusables_path = Path(__file__).parent.parent / CONFUSABLES_RS_PATH
confusables_path.write_text(formatted_data, encoding="utf-8")
print("Formatting Rust file with cargo fmt...")
subprocess.check_call(["cargo", "fmt", "--", confusables_path])
print("Done.")
if __name__ == "__main__":
main()