forked from jxmorris12/language_tool_python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmatch.py
More file actions
386 lines (320 loc) · 14.8 KB
/
Copy pathmatch.py
File metadata and controls
386 lines (320 loc) · 14.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
"""LanguageTool API Match object representation and utility module."""
import logging
import unicodedata
from collections import OrderedDict
from functools import total_ordering
from typing import (
Any,
Dict,
Iterator,
List,
Optional,
Tuple,
)
from typing import (
OrderedDict as OrderedDictType,
)
logger = logging.getLogger(__name__)
def get_match_ordered_dict() -> OrderedDictType[str, type]:
"""
Returns an ordered dictionary with predefined keys and their corresponding types.
:return: An OrderedDict where each key is a string representing a specific attribute
and each value is the type of that attribute.
:rtype: OrderedDictType[str, type]
The keys and their corresponding types are:
- 'rule_id': str
- 'message': str
- 'replacements': list
- 'offset_in_context': int
- 'context': str
- 'offset': int
- 'error_length': int
- 'category': str
- 'rule_issue_type': str
- 'sentence': str
"""
return OrderedDict(
[
("rule_id", str),
("message", str),
("replacements", list),
("offset_in_context", int),
("context", str),
("offset", int),
("error_length", int),
("category", str),
("rule_issue_type", str),
("sentence", str),
],
)
def auto_type(obj: Any) -> Any:
"""
Attempts to automatically convert the input object to an integer or float.
If the conversion to an integer fails, it tries to convert to a float.
If both conversions fail, it returns the original object.
:param obj: The object to be converted.
:type obj: Any
:return: The converted object as an integer, float, or the original object.
:rtype: Any
"""
try:
return int(obj)
except ValueError:
try:
return float(obj)
except ValueError:
return obj
def four_byte_char_positions(text: str) -> List[int]:
"""
Identify positions of 4-byte encoded characters in a UTF-8 string.
This function scans through the input text and identifies the positions
of characters that are encoded with 4 bytes in UTF-8. These characters
are typically non-BMP (Basic Multilingual Plane) characters, such as
certain emoji and some rare Chinese, Japanese, and Korean characters.
:param text: The input string to be analyzed.
:type text: str
:return: A list of positions where 4-byte encoded characters are found.
:rtype: List[int]
"""
positions: List[int] = []
char_index = 0
for char in text:
if len(char.encode("utf-8")) == 4:
positions.append(char_index)
# Adding 1 to the index because 4 byte characters are
# 2 bytes in length in LanguageTool, instead of 1 byte in Python.
char_index += 1
char_index += 1
if positions:
logger.debug("Found 4-byte encoded characters at positions: %s", positions)
return positions
@total_ordering
class Match:
"""
Represents a match object that contains information about a language rule violation.
:param attrib: A dictionary containing various attributes for the match.
The dictionary is expected to have the following keys:
- ``rule`` (*Dict[str, Any]*): A dictionary with keys ``category`` (which has an ``id``) and ``id``, ``issueType``.
- ``context`` (*Dict[str, Any]*): A dictionary with keys ``offset`` and ``text``.
- ``replacements`` (*List[Dict[str, str]]*): A list of dictionaries, each containing a ``value``.
- ``length`` (*int*): The length of the error.
- ``message`` (*str*): The message describing the error.
:type attrib: Dict[str, Any]
:param text: The original text in which the error occurred (the whole text, not just the context).
:type text: str
Example of a match object received from the LanguageTool API :
.. code-block:: python
{
'message': 'Possible spelling mistake found.',
'shortMessage': 'Spelling mistake',
'replacements': [{'value': 'newt'}, {'value': 'not'}, {'value': 'new', 'shortDescription': 'having just been made'}, {'value': 'news'}, {'value': 'foot', 'shortDescription': 'singular'}, {'value': 'root', 'shortDescription': 'underground organ of a plant'}, {'value': 'boot'}, {'value': 'noon'}, {'value': 'loot', 'shortDescription': 'plunder'}, {'value': 'moot'}, {'value': 'Root'}, {'value': 'soot', 'shortDescription': 'carbon black'}, {'value': 'newts'}, {'value': 'nook'}, {'value': 'Lieut'}, {'value': 'coot'}, {'value': 'hoot'}, {'value': 'toot'}, {'value': 'snoot'}, {'value': 'neut'}, {'value': 'nowt'}, {'value': 'Noor'}, {'value': 'noob'}],
'offset': 8,
'length': 4,
'context': {'text': 'This is noot okay. ', 'offset': 8, 'length': 4}, 'sentence': 'This is noot okay.',
'type': {'typeName': 'Other'},
'rule': {'id': 'MORFOLOGIK_RULE_EN_US', 'description': 'Possible spelling mistake', 'issueType': 'misspelling', 'category': {'id': 'TYPOS', 'name': 'Possible Typo'}},
'ignoreForIncompleteSentence': False,
'contextForSureMatch': 0
}
"""
PREVIOUS_MATCHES_TEXT: Optional[str] = None
"""The text of the previous match object."""
FOUR_BYTES_POSITIONS: Optional[List[int]] = None
"""The positions of 4-byte encoded characters in the text, registered by the previous match object
(kept for optimization purposes if the text is the same)."""
rule_id: str
"""The ID of the rule that was violated."""
message: str
"""The message describing the error."""
replacements: List[str]
"""A list of suggested replacements for the error."""
offset_in_context: int
"""The offset of the error in the context."""
context: str
"""The context in which the error occurred."""
offset: int
"""The offset of the error."""
error_length: int
"""The length of the error."""
category: str
"""The category of the rule that was violated."""
rule_issue_type: str
"""The issue type of the rule that was violated."""
def __init__(self, attrib: Dict[str, Any], text: str) -> None:
"""
Initialize a Match object with the given attributes.
The method processes and normalizes the attributes before storing them on the object.
This method adjusts the positions of 4-byte encoded characters in the text
to ensure the offsets of the matches are correct.
"""
# Process rule.
attrib["category"] = attrib["rule"]["category"]["id"]
attrib["rule_id"] = attrib["rule"]["id"]
attrib["rule_issue_type"] = attrib["rule"]["issueType"]
del attrib["rule"]
# Process context.
attrib["offset_in_context"] = attrib["context"]["offset"]
attrib["context"] = attrib["context"]["text"]
# Process replacements.
attrib["replacements"] = [r["value"] for r in attrib["replacements"]]
# Rename error length.
attrib["error_length"] = attrib["length"]
# Normalize unicode
attrib["message"] = unicodedata.normalize("NFKC", attrib["message"])
# Store objects on self.
for k, v in attrib.items():
setattr(self, k, v)
if text != Match.PREVIOUS_MATCHES_TEXT:
Match.PREVIOUS_MATCHES_TEXT = text
Match.FOUR_BYTES_POSITIONS = four_byte_char_positions(text)
# Get the positions of 4-byte encoded characters in the text because without
# carrying out this step, the offsets of the matches could be incorrect.
if Match.FOUR_BYTES_POSITIONS is not None:
self.offset -= sum(
1 for pos in Match.FOUR_BYTES_POSITIONS if pos < self.offset
)
def __repr__(self) -> str:
"""
Return a string representation of the object.
This method provides a detailed string representation of the object,
including its class name and a dictionary of its attributes.
:return: A string representation of the object.
:rtype: str
"""
def _ordered_dict_repr() -> str:
"""
Generate a string representation of the object's attributes in an ordered dictionary format.
This method collects the attributes of the object, ensuring that the order of attributes
is preserved as defined by ``get_match_ordered_dict()``. Attributes that are not part of the
ordered dictionary are appended at the end. Attributes starting with an underscore are
excluded from the representation.
:return: A string representation of the object's attributes in an ordered dictionary format.
:rtype: str
"""
slots = list(get_match_ordered_dict())
slots += list(set(self.__dict__).difference(slots))
attrs = [
slot
for slot in slots
if slot in self.__dict__ and not slot.startswith("_")
]
return f"{{{', '.join([f'{attr!r}: {getattr(self, attr)!r}' for attr in attrs])}}}"
return f"{self.__class__.__name__}({_ordered_dict_repr()})"
def __str__(self) -> str:
"""
Returns a string representation of the match object.
The string includes the offset, error length, rule ID, message,
suggestions, and context with a visual indicator of the error position.
:return: A formatted string describing the match object.
:rtype: str
"""
rule_id = self.rule_id
s = f"Offset {self.offset}, length {self.error_length}, Rule ID: {rule_id}"
if self.message:
s += f"\nMessage: {self.message}"
if self.replacements:
s += f"\nSuggestion: {'; '.join(self.replacements)}"
s += f"\n{self.context}\n{' ' * self.offset_in_context + '^' * self.error_length}"
return s
@property
def matched_text(self) -> str:
"""
Returns the substring from the context that corresponds to the matched text.
:return: The matched text from the context.
:rtype: str
"""
return self.context[
self.offset_in_context : self.offset_in_context + self.error_length
]
def get_line_and_column(self, original_text: str) -> Tuple[int, int]:
"""
Returns the line and column number of the error in the context.
:param original_text: The original text in which the error occurred. We need this to calculate the line and column number, because the context has no more newline characters.
:type original_text: str
:return: A tuple containing the line and column number of the error.
:rtype: Tuple[int, int]
"""
context_without_additions = (
self.context[3:-3] if len(self.context) > 6 else self.context
)
if context_without_additions not in original_text.replace("\n", " "):
err = "The original text does not match the context of the error"
raise ValueError(err)
line = original_text.count("\n", 0, self.offset)
column = self.offset - original_text.rfind("\n", 0, self.offset)
return line + 1, column
def select_replacement(self, index: int) -> None:
"""
Select a single replacement suggestion based on the given index and update the replacements list, leaving only the selected replacement.
:param index: The index of the replacement to select.
:type index: int
:raises ValueError: If there are no replacement suggestions.
:raises ValueError: If the index is out of the valid range.
"""
if not self.replacements:
err = "This Match has no suggestions"
raise ValueError(err)
if index < 0 or index >= len(self.replacements):
err = f"This Match's suggestions are numbered from 0 to {len(self.replacements) - 1}"
raise ValueError(err)
self.replacements = [self.replacements[index]]
def __eq__(self, other: Any) -> bool:
"""
Compare this object with another for equality.
:param other: The object to compare with.
:type other: Any
:return: True if both objects are equal, False otherwise.
:rtype: bool
"""
return list(self) == list(other)
def __lt__(self, other: Any) -> bool:
"""
Compare this object with another object for less-than ordering.
:param other: The object to compare with.
:type other: Any
:return: True if this object is less than the other object, False otherwise.
:rtype: bool
"""
return list(self) < list(other)
def __iter__(self) -> Iterator[Any]:
"""
Return an iterator over the attributes of the match object.
This method allows the match object to be iterated over, yielding the
values of its attributes in the order defined by ``get_match_ordered_dict()``.
:return: An iterator over the attribute values of the match object.
:rtype: Iterator[Any]
"""
return iter(getattr(self, attr) for attr in get_match_ordered_dict())
def __setattr__(self, key: str, value: Any) -> None:
"""
Set an attribute on the instance.
This method overrides the default behavior of setting an attribute.
It attempts to transform the value using a function from ``get_match_ordered_dict()``
based on the provided key. If the key is not found in the dictionary, the attribute
is not set.
:param key: The name of the attribute to set.
:type key: str
:param value: The value to set the attribute to.
:type value: Any
"""
try:
value = get_match_ordered_dict()[key](value)
except KeyError:
return
super().__setattr__(key, value)
def __getattr__(self, name: str) -> Any:
"""
Handle attribute access for undefined attributes.
This method is called when an attribute lookup has not found the attribute in the usual places
(i.e., it is not an instance attribute nor is it found in the class tree for self). This method
checks if the attribute name is in the ordered dictionary returned by ``get_match_ordered_dict()``.
If the attribute name is not found, it raises an AttributeError.
:param name: The name of the attribute being accessed.
:type name: str
:return: The value of the attribute if it exists.
:rtype: Any
:raises AttributeError: If the attribute does not exist.
"""
if name not in get_match_ordered_dict():
err = f"{self.__class__.__name__!r} object has no attribute {name!r}"
raise AttributeError(err)