forked from jxmorris12/language_tool_python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmatch.py
More file actions
453 lines (381 loc) · 16.9 KB
/
Copy pathmatch.py
File metadata and controls
453 lines (381 loc) · 16.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
"""LanguageTool API Match object representation and utility module."""
from __future__ import annotations
import logging
import typing
import unicodedata
from collections import OrderedDict
from collections import OrderedDict as OrderedDictType
from functools import total_ordering
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from collections.abc import Iterator
from ._internals.api_types import CheckMatch
__all__ = ["Match", "four_byte_char_positions", "is_check_match"]
logger = logging.getLogger(__name__)
_UTF8_4_BYTE_LENGTH = 4
_CONTEXT_PREFIX_SUFFIX_LENGTH = 3
_CONTEXT_WITH_ADDITIONS_MIN_LENGTH = 6
_MatchValue = str | int | list[str]
def _get_match_ordered_dict() -> OrderedDictType[str, type]:
"""Return an ordered dictionary with predefined keys and their corresponding types.
:return: An OrderedDict where each key is a string representing a specific attribute
and each value is the type of that attribute.
:rtype: OrderedDictType[str, type]
The keys and their corresponding types are:
- 'rule_id': str
- 'message': str
- 'replacements': list
- 'offset_in_context': int
- 'context': str
- 'offset': int
- 'error_length': int
- 'category': str
- 'rule_issue_type': str
- 'sentence': str
"""
return OrderedDict(
[
("rule_id", str),
("message", str),
("replacements", list),
("offset_in_context", int),
("context", str),
("offset", int),
("error_length", int),
("category", str),
("rule_issue_type", str),
("sentence", str),
],
)
def four_byte_char_positions(text: str) -> list[int]:
"""Identify positions of 4-byte encoded characters in a UTF-8 string.
This function scans through the input text and identifies the positions of
characters that are encoded with 4 bytes in UTF-8. These characters are typically
non-BMP (Basic Multilingual Plane) characters, such as certain emoji and some rare
Chinese, Japanese, and Korean characters.
:param text: The input string to be analyzed.
:type text: str
:return: A list of positions where 4-byte encoded characters are found.
:rtype: list[int]
"""
positions: list[int] = []
char_index = 0
for char in text:
if len(char.encode("utf-8")) == _UTF8_4_BYTE_LENGTH:
positions.append(char_index)
# Adding 1 to the index because 4 byte characters are
# 2 bytes in length in LanguageTool, instead of 1 byte in Python.
char_index += 1
char_index += 1
if positions:
logger.debug("Found 4-byte encoded characters at positions: %s", positions)
return positions
def is_check_match(value: object) -> typing.TypeGuard[CheckMatch]:
"""Verify that a value is a CheckMatch.
:param value: The value to check.
:type value: object
:return: TypeGuard indicating whether the value is a CheckMatch.
:rtype: typing.TypeGuard[CheckMatch]
"""
if not isinstance(value, dict):
return False
return (
isinstance(value.get("message"), str)
and isinstance(value.get("shortMessage"), str)
and isinstance(value.get("replacements"), list)
and isinstance(value.get("offset"), int)
and isinstance(value.get("length"), int)
and isinstance(value.get("context"), dict)
and isinstance(value.get("sentence"), str)
and isinstance(value.get("type"), dict)
and isinstance(value.get("rule"), dict)
and isinstance(value.get("ignoreForIncompleteSentence"), bool)
and isinstance(value.get("contextForSureMatch"), int)
)
@total_ordering
class Match: # noqa: PLW1641 # Doesn't implement hash because it's mutable
"""Represent a language rule violation match.
:param attrib: A raw LanguageTool API match. It is expected to contain ``rule``
(with ``category``, ``id``, and ``issueType``), ``context`` (with ``offset``
and ``text``), ``replacements`` (items with ``value``), ``length``, and
``message``.
:type attrib: CheckMatch
:param four_byte_positions: The positions of 4-byte encoded characters in the
original text in which the error occurred (the whole text, not just the
context), as returned by :func:`four_byte_char_positions`.
:type four_byte_positions: list[int]
Example of a match object received from the LanguageTool API :
.. code-block:: python
{
"message": "Possible spelling mistake found.",
"shortMessage": "Spelling mistake",
"replacements": [
{"value": "newt"},
{"value": "not"},
{"value": "new", "shortDescription": "having just been made"},
{"value": "news"},
{"value": "foot", "shortDescription": "singular"},
{"value": "root", "shortDescription": "underground organ of a plant"},
{"value": "boot"},
{"value": "noon"},
{"value": "loot", "shortDescription": "plunder"},
{"value": "moot"},
{"value": "Root"},
{"value": "soot", "shortDescription": "carbon black"},
{"value": "newts"},
{"value": "nook"},
{"value": "Lieut"},
{"value": "coot"},
{"value": "hoot"},
{"value": "toot"},
{"value": "snoot"},
{"value": "neut"},
{"value": "nowt"},
{"value": "Noor"},
{"value": "noob"},
],
"offset": 8,
"length": 4,
"context": {"text": "This is noot okay. ", "offset": 8, "length": 4},
"sentence": "This is noot okay.",
"type": {"typeName": "Other"},
"rule": {
"id": "MORFOLOGIK_RULE_EN_US",
"description": "Possible spelling mistake",
"issueType": "misspelling",
"category": {"id": "TYPOS", "name": "Possible Typo"},
},
"ignoreForIncompleteSentence": False,
"contextForSureMatch": 0,
}
"""
rule_id: str
"""The ID of the rule that was violated."""
message: str
"""The message describing the error."""
replacements: list[str]
"""A list of suggested replacements for the error."""
offset_in_context: int
"""The offset of the error in the context."""
context: str
"""The context in which the error occurred."""
offset: int
"""The offset of the error."""
error_length: int
"""The length of the error."""
category: str
"""The category of the rule that was violated."""
rule_issue_type: str
"""The issue type of the rule that was violated."""
sentence: str
"""The sentence that contains the rule violation."""
def __init__(
self,
attrib: CheckMatch,
four_byte_positions: list[int],
) -> None:
"""Initialize a Match object with the given attributes.
The method processes and normalizes the attributes before storing them on the
object. This method adjusts the positions of 4-byte encoded characters in the
text to ensure the offsets of the matches are correct.
:param attrib: The raw LanguageTool API match.
:type attrib: CheckMatch
:param four_byte_positions: The positions of 4-byte encoded characters in the
original text (the whole text, not just the context), as returned by
:func:`four_byte_char_positions`. Callers processing multiple matches for
the same text should compute this once and reuse it across all matches.
:type four_byte_positions: list[int]
"""
# Process rule.
custom_match: dict[str, str | int | list[str]] = {}
custom_match["category"] = attrib["rule"]["category"]["id"]
custom_match["rule_id"] = attrib["rule"]["id"]
custom_match["rule_issue_type"] = attrib["rule"]["issueType"]
# Process context.
custom_match["offset_in_context"] = attrib["context"]["offset"]
custom_match["context"] = attrib["context"]["text"]
# Process replacements.
custom_match["replacements"] = [r["value"] for r in attrib["replacements"]]
# Rename error length.
custom_match["error_length"] = attrib["length"]
# Normalize unicode
custom_match["message"] = unicodedata.normalize("NFKC", attrib["message"])
custom_match["sentence"] = attrib["sentence"]
# Store offset before adjusting it for 4-byte characters
custom_match["offset"] = attrib["offset"]
# Store objects on self.
for k, v in custom_match.items():
setattr(self, k, v)
# Adjust the offset for 4-byte encoded characters because without carrying out
# this step, the offsets of the matches could be incorrect.
offset = self.offset
adjustment = 0
for pos in four_byte_positions:
if pos >= offset:
break
adjustment += 1
self.offset = offset - adjustment
def _ordered_items(self) -> list[tuple[str, _MatchValue]]:
"""Return public match attributes in the documented order."""
return [
("rule_id", self.rule_id),
("message", self.message),
("replacements", self.replacements),
("offset_in_context", self.offset_in_context),
("context", self.context),
("offset", self.offset),
("error_length", self.error_length),
("category", self.category),
("rule_issue_type", self.rule_issue_type),
("sentence", self.sentence),
]
def __repr__(self) -> str:
"""Return a string representation of the object.
This method provides a detailed string representation of the object, including
its class name and a dictionary of its attributes.
:return: A string representation of the object.
:rtype: str
"""
def _ordered_dict_repr() -> str:
"""Return the object's attributes as an ordered dictionary string.
This method collects the attributes of the object, ensuring that the order
of attributes is preserved as defined by ``get_match_ordered_dict()``.
Attributes that are not part of the ordered dictionary are appended at the
end. Attributes starting with an underscore are excluded from the
representation.
:return: A string representation of the object's attributes in an ordered
dictionary format.
:rtype: str
"""
items = ", ".join(
f"{attr!r}: {value!r}" for attr, value in self._ordered_items()
)
return f"{{{items}}}"
return f"{self.__class__.__name__}({_ordered_dict_repr()})"
def __str__(self) -> str:
"""Return a string representation of the match object.
The string includes the offset, error length, rule ID, message, suggestions, and
context with a visual indicator of the error position.
:return: A formatted string describing the match object.
:rtype: str
"""
rule_id = self.rule_id
s = f"Offset {self.offset}, length {self.error_length}, Rule ID: {rule_id}"
if self.message:
s += f"\nMessage: {self.message}"
if self.replacements:
s += f"\nSuggestion: {'; '.join(self.replacements)}"
s += (
f"\n{self.context}\n"
f"{' ' * self.offset_in_context + '^' * self.error_length}"
)
return s
@property
def matched_text(self) -> str:
"""Return the substring from the context that corresponds to the matched text.
:return: The matched text from the context.
:rtype: str
"""
return self.context[
self.offset_in_context : self.offset_in_context + self.error_length
]
def get_line_and_column(self, original_text: str) -> tuple[int, int]:
"""Return the line and column number of the error in the context.
:param original_text: The original text in which the error occurred. We need
this to calculate the line and column number, because the context has no
more newline characters.
:type original_text: str
:return: A tuple containing the line and column number of the error.
:rtype: tuple[int, int]
:raises ValueError: If the original text does not contain the match context.
"""
context_without_additions = (
self.context[_CONTEXT_PREFIX_SUFFIX_LENGTH:-_CONTEXT_PREFIX_SUFFIX_LENGTH]
if len(self.context) > _CONTEXT_WITH_ADDITIONS_MIN_LENGTH
else self.context
)
if context_without_additions not in original_text.replace("\n", " "):
err = "The original text does not match the context of the error"
raise ValueError(err)
line = original_text.count("\n", 0, self.offset)
column = self.offset - original_text.rfind("\n", 0, self.offset)
return line + 1, column
def select_replacement(self, index: int) -> None:
"""Keep only the replacement selected by the given index.
:param index: The index of the replacement to select.
:type index: int
:raises ValueError: If there are no replacement suggestions.
:raises ValueError: If the index is out of the valid range.
"""
if not self.replacements:
err = "This Match has no suggestions"
raise ValueError(err)
if index < 0 or index >= len(self.replacements):
err = (
f"This Match's suggestions are numbered from 0"
f"to {len(self.replacements) - 1}"
)
raise ValueError(err)
self.replacements = [self.replacements[index]]
def __eq__(self, other: object) -> bool:
"""Compare this object with another for equality.
:param other: The object to compare with.
:type other: object
:return: True if both objects are equal, False otherwise.
:rtype: bool
"""
if not isinstance(other, Match):
return NotImplemented
return list(self) == list(other)
def __lt__(self, other: object) -> bool:
"""Compare this object with another object for less-than ordering.
:param other: The object to compare with.
:type other: object
:return: True if this object is less than the other object, False otherwise.
:rtype: bool
"""
if not isinstance(other, Match):
return NotImplemented
return list(self) < list(other)
def __iter__(self) -> Iterator[_MatchValue]:
"""Return an iterator over the attributes of the match object.
This method allows the match object to be iterated over, yielding the values of
its attributes in the order defined by ``get_match_ordered_dict()``.
:return: An iterator over the attribute values of the match object.
:rtype: Iterator[str | int | list[str]]
"""
return iter(value for _, value in self._ordered_items())
def __setattr__(self, key: str, value: _MatchValue) -> None:
"""Set an attribute on the instance.
This method overrides the default behavior of setting an attribute. It attempts
to transform the value using a function from ``get_match_ordered_dict()`` based
on the provided key. If the key is not found in the dictionary, the attribute is
not set.
:param key: The name of the attribute to set.
:type key: str
:param value: The value to set the attribute to.
:type value: str | int | list[str]
"""
try:
# Ex: if key is "offset", get_match_ordered_dict()[key] will return int, so
# the value will be transformed to int
value = _get_match_ordered_dict()[key](value)
except KeyError:
return
super().__setattr__(key, value)
def __getattr__(self, name: str) -> None:
"""Handle attribute access for undefined attributes.
This method is called when an attribute lookup has not found the attribute in
the usual places (i.e., it is not an instance attribute nor is it found in the
class tree for self). This method checks if the attribute name is in the ordered
dictionary returned by ``get_match_ordered_dict()``. If the attribute name is
not found, it raises an AttributeError.
:param name: The name of the attribute being accessed.
:type name: str
:return: None for known unset match fields.
:rtype: None
:raises AttributeError: If the attribute does not exist.
"""
if name not in _get_match_ordered_dict():
err = f"{self.__class__.__name__!r} object has no attribute {name!r}"
raise AttributeError(err)