forked from jxmorris12/language_tool_python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathconfig_file.py
More file actions
193 lines (171 loc) · 10.3 KB
/
Copy pathconfig_file.py
File metadata and controls
193 lines (171 loc) · 10.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
import os
import atexit
import tempfile
import multiprocessing as mp
from typing import Any, Dict, Optional
from pydantic import BaseSettings, Field
from language_tool_python.logs import logger
ALLOWED_CONFIG_KEYS = {
'maxTextLength',
'maxTextHardLength',
'secretTokenKey',
'maxCheckTimeMillis',
'maxErrorsPerWordRate',
'maxSpellingSuggestions',
'maxCheckThreads',
'cacheTTLSeconds',
'cacheSize',
'requestLimit',
'requestLimitInBytes',
'timeoutRequestLimit',
'requestLimitPeriodInSeconds',
'languageModel',
'word2vecModel',
'fasttextModel',
'fasttextBinary',
'maxWorkQueueSize',
'rulesFile',
'warmUp',
'blockedReferrers',
'premiumOnly',
'disabledRuleIds',
'pipelineCaching',
'maxPipelinePoolSize',
'pipelineExpireTimeInSeconds',
'pipelinePrewarming'
}
def to_camel_case(text: str):
"""Convert a snake str to camel case."""
components = text.split("_")
# We capitalize the first letter of each component except the first one
# with the 'title' method and join them together.
return components[0] + "".join(x.title() for x in components[1:])
def get_max_check_threads():
"""Get the maximum number of threads to use for checking."""
return mp.cpu_count() * 4
class ServerConfig(BaseSettings):
"""LT Server Configurations."""
max_text_length: int = Field(0, alias='maxTextLength')
max_text_hard_length: int = Field(0, alias='maxTextHardLength')
secret_token_key: str = Field('', alias='secretTokenKey')
max_check_time_millis: int = Field(0, alias='maxCheckTimeMillis')
max_errors_per_word_rate: int = Field(0, alias='maxErrorsPerWordRate')
max_spelling_suggestions: int = Field(0, alias='maxSpellingSuggestions')
max_check_threads: int = Field(default_factory = get_max_check_threads, alias='maxCheckThreads')
cache_ttl_seconds: int = Field(300, alias='cacheTTLSeconds')
cache_size: int = Field(1000, alias='cacheSize')
request_limit: int = Field(100, alias='requestLimit')
request_limit_in_bytes: int = Field(1000000, alias='requestLimitInBytes')
timeout_request_limit: int = Field(100, alias='timeoutRequestLimit')
request_limit_period_in_seconds: int = Field(60, alias='requestLimitPeriodInSeconds')
language_model: str = Field('', alias='languageModel')
word2vec_model: str = Field('', alias='word2vecModel')
fasttext_model: str = Field('', alias='fasttextModel')
fasttext_binary: str = Field('', alias='fasttextBinary')
max_work_queue_size: int = Field(1000, alias='maxWorkQueueSize')
rules_file: str = Field('', alias='rulesFile')
warm_up: bool = Field(True, alias='warmUp')
blocked_referrers: str = Field('', alias='blockedReferrers')
premium_only: bool = Field(False, alias='premiumOnly')
disabled_rule_ids: str = Field('', alias='disabledRuleIds')
pipeline_caching: bool = Field(True, alias='pipelineCaching')
max_pipeline_pool_size: int = Field(100, alias='maxPipelinePoolSize')
pipeline_expire_time_in_seconds: int = Field(60, alias='pipelineExpireTimeInSeconds')
pipeline_prewarming: bool = Field(False, alias='pipelinePrewarming')
# Other Settings
public: bool = True
premium_always: bool = True
class Config:
env_prefix = 'LT_'
case_sensitive = False
def to_config(self) -> Dict[str, Any]:
"""Convert config to dict."""
config = self.dict(
by_alias = True,
exclude_none = True,
exclude = {'public', 'premium_always'},
)
config = {k: v for k,v in config.items() if v}
return config
def get_server_options(self):
"""Get server options."""
options = []
if self.public: options.append('--public')
if self.premium_always: options.append('--premium-always')
return options
class LanguageToolConfig:
config: Dict[str, Any]
path: str
def __init__(self, config: Dict[str, Any]):
logger.info(f'Creating config file with {config}')
assert set(config.keys()) <= ALLOWED_CONFIG_KEYS, f"unexpected keys in config: {set(config.keys()) - ALLOWED_CONFIG_KEYS}"
assert len(config), "config cannot be empty"
self.config = config
self.path = self._create_temp_file()
def _create_temp_file(self) -> str:
tmp_file = tempfile.NamedTemporaryFile(delete=False)
# WRite key=value entries as lines in temporary file.
for key, value in self.config.items():
next_line = f'{key}={value}\n'
tmp_file.write(next_line.encode())
tmp_file.close()
# Remove file when program exits.
atexit.register(lambda: os.unlink(tmp_file.name))
return tmp_file.name
"""
❯ /usr/bin/java -cp /Users/johnmorris/.cache/language_tool_python/LanguageTool-5.6/languagetool-server.jar org.languagetool.server.HTTPServer --help
Usage: HTTPServer [--config propertyFile] [--port|-p port] [--public]
--config FILE a Java property file (one key=value entry per line) with values for:
'maxTextLength' - maximum text length, longer texts will cause an error (optional)
'maxTextHardLength' - maximum text length, applies even to users with a special secret 'token' parameter (optional)
'secretTokenKey' - secret JWT token key, if set by user and valid, maxTextLength can be increased by the user (optional)
'maxCheckTimeMillis' - maximum time in milliseconds allowed per check (optional)
'maxErrorsPerWordRate' - checking will stop with error if there are more rules matches per word (optional)
'maxSpellingSuggestions' - only this many spelling errors will have suggestions for performance reasons (optional,
affects Hunspell-based languages only)
'maxCheckThreads' - maximum number of threads working in parallel (optional)
'cacheSize' - size of internal cache in number of sentences (optional, default: 0)
'cacheTTLSeconds' - how many seconds sentences are kept in cache (optional, default: 300 if 'cacheSize' is set)
'requestLimit' - maximum number of requests per requestLimitPeriodInSeconds (optional)
'requestLimitInBytes' - maximum aggregated size of requests per requestLimitPeriodInSeconds (optional)
'timeoutRequestLimit' - maximum number of timeout request (optional)
'requestLimitPeriodInSeconds' - time period to which requestLimit and timeoutRequestLimit applies (optional)
'languageModel' - a directory with '1grams', '2grams', '3grams' sub directories which contain a Lucene index
each with ngram occurrence counts; activates the confusion rule if supported (optional)
'word2vecModel' - a directory with word2vec data (optional), see
https://github.com/languagetool-org/languagetool/blob/master/languagetool-standalone/CHANGES.md#word2vec
'fasttextModel' - a model file for better language detection (optional), see
https://fasttext.cc/docs/en/language-identification.html
'fasttextBinary' - compiled fasttext executable for language detection (optional), see
https://fasttext.cc/docs/en/support.html
'maxWorkQueueSize' - reject request if request queue gets larger than this (optional)
'rulesFile' - a file containing rules configuration, such as .langugagetool.cfg (optional)
'warmUp' - set to 'true' to warm up server at start, i.e. run a short check with all languages (optional)
'blockedReferrers' - a comma-separated list of HTTP referrers (and 'Origin' headers) that are blocked and will not be served (optional)
'premiumOnly' - activate only the premium rules (optional)
'disabledRuleIds' - a comma-separated list of rule ids that are turned off for this server (optional)
'pipelineCaching' - set to 'true' to enable caching of internal pipelines to improve performance
'maxPipelinePoolSize' - cache size if 'pipelineCaching' is set
'pipelineExpireTimeInSeconds' - time after which pipeline cache items expire
'pipelinePrewarming' - set to 'true' to fill pipeline cache on start (can slow down start a lot)
Spellcheck-only languages: You can add simple spellcheck-only support for languages that LT doesn't
support by defining two optional properties:
'lang-xx' - set name of the language, use language code instead of 'xx', e.g. lang-tr=Turkish
'lang-xx-dictPath' - absolute path to the hunspell .dic file, use language code instead of 'xx', e.g.
lang-tr-dictPath=/path/to/tr.dic. Note that the same directory also needs to
contain a common_words.txt file with the most common 10,000 words (used for better language detection)
--port, -p PRT port to bind to, defaults to 8081 if not specified
--public allow this server process to be connected from anywhere; if not set,
it can only be connected from the computer it was started on
--allow-origin [ORIGIN] set the Access-Control-Allow-Origin header in the HTTP response,
used for direct (non-proxy) JavaScript-based access from browsers.
Example: --allow-origin "https://my-website.org"
Don't set a parameter for `*`, i.e. access from all websites.
--verbose, -v in case of exceptions, log the input text (up to 500 characters)
--languageModel a directory with '1grams', '2grams', '3grams' sub directories (per language)
which contain a Lucene index (optional, overwrites 'languageModel'
parameter in properties files)
--word2vecModel a directory with word2vec data (optional), see
https://github.com/languagetool-org/languagetool/blob/master/languagetool-standalone/CHANGES.md#word2vec
--premiumAlways activate the premium rules even when user has no username/password - useful for API servers
"""