Skip to content

Commit 9119479

Browse files
dhondtadhondta
committed
Refactored __common__
1 parent 1fa82c1 commit 9119479

4 files changed

Lines changed: 248 additions & 60 deletions

File tree

‎codext/__common__.py‎

Lines changed: 120 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,57 @@
11
# -*- coding: UTF-8 -*-
22
import _codecs
33
import codecs
4+
import os
45
import re
56
import sys
6-
import types
77
from functools import wraps
88
from six import binary_type, string_types, text_type
9+
from types import FunctionType
10+
try: # Python3
11+
from inspect import getfullargspec
12+
except ImportError:
13+
from inspect import getargspec as getfullargspec
14+
try:
15+
from importlib import reload
16+
except ImportError:
17+
pass
918

1019

11-
__all__ = ["add", "b", "codecs", "ensure_str", "re", "PY3"]
20+
__all__ = ["add", "b", "clear", "codecs", "ensure_str", "re", "register",
21+
"remove", "reset", "s2i", "PY3"]
1222
PY3 = sys.version[0] == "3"
1323

1424

1525
isb = lambda s: isinstance(s, binary_type)
1626
iss = lambda s: isinstance(s, string_types)
1727
fix = lambda x, ref: b(x) if isb(ref) else ensure_str(x) if iss(ref) else x
1828

29+
s2i = lambda s: int(codecs.encode(s, "base16"), 16)
1930

20-
def add(ename, encode=None, decode=None, pattern=None, text=True):
31+
32+
def add(ename, encode=None, decode=None, pattern=None, text=True,
33+
add_to_codecs=False):
2134
"""
2235
This adds a new codec to the codecs module setting its encode and/or decode
2336
functions, eventually dynamically naming the encoding with a pattern and
2437
with file handling (if text is True).
2538
26-
:param ename: encoding name
27-
:param encode: encoding function or None
28-
:param decode: decoding function or None
29-
:param pattern: pattern for dynamically naming the encoding
30-
:param text: specify whether the codec is a text encoding
31-
"""
32-
if encode and not isinstance(encode, types.FunctionType):
33-
raise ValueError("Bad encode function")
34-
if decode and not isinstance(decode, types.FunctionType):
35-
raise ValueError("Bad decode function")
39+
:param ename: encoding name
40+
:param encode: encoding function or None
41+
:param decode: decoding function or None
42+
:param pattern: pattern for dynamically naming the encoding
43+
:param text: specify whether the codec is a text encoding
44+
:param add_to_codecs: also add the search function to the native registry
45+
NB: this will make the codec available in the
46+
built-in open(...) but will make it impossible
47+
to remove the codec later
48+
"""
49+
if encode and not isinstance(encode, FunctionType):
50+
raise ValueError("Bad 'encode' function")
51+
if decode and not isinstance(decode, FunctionType):
52+
raise ValueError("Bad 'decode' function")
3653
if not encode and not decode:
37-
raise ValueError("At least one function must be defined")
54+
raise ValueError("At least one en/decoding function must be defined")
3855
# search function for the new encoding
3956
def getregentry(encoding):
4057
if encoding != ename and not (pattern and re.match(pattern, encoding)):
@@ -79,8 +96,13 @@ def decode(self, input, final=False):
7996
except AttributeError:
8097
return # this occurs when m is None, meaning no match
8198
except IndexError:
82-
pass # this occurs while m is not None, but possibly no
83-
# capture group that gives at least 1 group index
99+
# this occurs while m is not None, but possibly no capture group
100+
# that gives at least 1 group index ; in this case, if
101+
# fenc/fdec is a decorated function, execute it with no arg
102+
if fenc and len(getfullargspec(fenc).args) == 1:
103+
fenc = fenc()
104+
if fdec and len(getfullargspec(fdec).args) == 1:
105+
fdec = fdec()
84106
if fenc:
85107
fenc = fix_inout_formats(fenc)
86108
if fdec:
@@ -107,21 +129,59 @@ class StreamReader(Codec, codecs.StreamReader):
107129
streamreader=streamreader,
108130
_is_text_encoding=text,
109131
)
110-
codecs.register(getregentry)
111-
codecs.add = add
132+
getregentry.__name__ = re.sub(r"[\s\-]", "_", ename)
133+
register(getregentry, add_to_codecs)
134+
135+
136+
def clear():
137+
"""
138+
Clear codext's local registry of search functions.
139+
"""
140+
global __codecs_registry
141+
__codecs_registry = []
142+
codecs.clear = clear
143+
144+
145+
def remove(encoding):
146+
"""
147+
Remove all search functions matching the input encoding name from codext's
148+
local registry.
149+
150+
:param encoding: encoding name
151+
"""
152+
tbr = []
153+
for search in __codecs_registry:
154+
if search(encoding) is not None:
155+
tbr.append(search)
156+
for search in tbr:
157+
__codecs_registry.remove(search)
158+
codecs.remove = remove
159+
160+
161+
def reset():
162+
"""
163+
Reset codext's local registry of search functions.
164+
"""
165+
clear()
166+
for f in os.listdir(os.path.dirname(__file__)):
167+
if not f.endswith(".py") or f.startswith("_"):
168+
continue
169+
reload(__import__(f[:-3], globals(), locals(), [], 1))
170+
codecs.reset = reset
112171

113172

173+
# conversion functions
114174
def b(s):
115175
"""
116176
Non-crashing bytes conversion function.
117177
"""
118178
if PY3:
119179
try:
120-
return s.encode("latin-1")
180+
return s.encode("utf-8")
121181
except:
122182
pass
123183
try:
124-
return s.encode("utf-8")
184+
return s.encode("latin-1")
125185
except:
126186
pass
127187
return s
@@ -159,27 +219,30 @@ def _wrapper(*args, **kwargs):
159219

160220

161221
# codecs module hooks
162-
orig_lookup = _codecs.lookup
163-
orig_register = _codecs.register
164-
_ts_codecs_registry = []
165-
_ts_codecs_registry_hashes = []
222+
orig_lookup = _codecs.lookup
223+
orig_register = _codecs.register
224+
225+
226+
def __add(ename, encode=None, decode=None, pattern=None, text=True,
227+
add_to_codecs=True):
228+
add(ename, encode, decode, pattern, text, add_to_codecs)
229+
__add.__doc__ = add.__doc__
230+
codecs.add = __add
166231

167232

168233
def __decode(obj, encoding='utf-8', errors='strict'):
169234
"""
170235
Custom decode function relying on the hooked lookup function.
171236
"""
172-
codecinfo = __lookup(encoding)
173-
return codecinfo.decode(obj, errors)[0]
237+
return __lookup(encoding).decode(obj, errors)[0]
174238
codecs.decode = __decode
175239

176240

177241
def __encode(obj, encoding='utf-8', errors='strict'):
178242
"""
179243
Custom encode function relying on the hooked lookup function.
180244
"""
181-
codecinfo = __lookup(encoding)
182-
return codecinfo.encode(obj, errors)[0]
245+
return __lookup(encoding).encode(obj, errors)[0]
183246
codecs.encode = __encode
184247

185248

@@ -188,22 +251,43 @@ def __lookup(encoding):
188251
Hooked lookup function for searching first for codecs in the local registry
189252
of this module.
190253
"""
191-
for search in _ts_codecs_registry:
254+
for search in __codecs_registry:
192255
codecinfo = search(encoding)
193256
if codecinfo is not None:
194257
return codecinfo
195258
return orig_lookup(encoding)
196259
codecs.lookup = __lookup
197260

198261

199-
def __register(search_function):
262+
def register(search_function, add_to_codecs=False):
263+
"""
264+
Register function for registering new codecs in the local registry of this
265+
module and, if required, in the native codecs registry (for use with the
266+
built-in 'open' function).
267+
268+
:param search_function: search function for the codecs registry
269+
:param add_to_codecs: also add the search function to the native registry
270+
NB: this will make the codec available in the
271+
built-in open(...) but will make it impossible
272+
to remove the codec later
273+
"""
274+
if search_function not in __codecs_registry:
275+
__codecs_registry.append(search_function)
276+
if add_to_codecs:
277+
orig_register(search_function)
278+
279+
280+
def __register(search_function, add_to_codecs=True):
200281
"""
201282
Hooked register function for registering new codecs in the local registry
202-
of this module.
283+
of this module and in the native codecs registry (for use with the built-in
284+
'open' function).
285+
286+
:param search_function: search function for the codecs registry
287+
:param add_to_codecs: also add the search function to the native registry
288+
NB: this will make the codec available in the
289+
built-in open(...) but will make it impossible
290+
to remove the codec later
203291
"""
204-
h = hash(search_function)
205-
if h not in _ts_codecs_registry_hashes:
206-
_ts_codecs_registry_hashes.append(h)
207-
_ts_codecs_registry.append(search_function)
208-
orig_register(search_function)
292+
register(search_function, add_to_codecs)
209293
codecs.register = __register

‎codext/__init__.py‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,21 @@
22
"""Module for enhancing codecs preimport.
33
44
"""
5-
import os
5+
from .__common__ import *
6+
from .__info__ import __author__, __copyright__, __license__, __version__
67

7-
from .__common__ import add, codecs
88

9-
10-
__all__ = ["add", "decode", "encode", "lookup", "open", "register"]
9+
__all__ = ["add", "clear", "decode", "encode", "lookup", "open", "register",
10+
"remove", "reset"]
1111

1212

1313
decode = codecs.decode
1414
encode = codecs.encode
1515
lookup = codecs.lookup
1616
open = codecs.open
17-
register = codecs.register
1817

1918

20-
for f in os.listdir(os.path.dirname(__file__)):
21-
if not f.endswith(".py") or f == "__init__.py":
22-
continue
23-
__import__(f[:-3], globals(), locals(), [], 1)
19+
reset()
2420

2521

2622
def main():

‎docs/features.md‎

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
1-
Basically, the `codecs` library, relying on the built-in `_codecs` library, maintains a registry of search functions that maps an input `encoding` variable to the right de/encode function. `codext` hooks the native `codecs` to insert its own registry between the function calls and the native one.
1+
Basically, the `codecs` library provides a series of functions from the built-in `_codecs` library which maintains a registry of search functions (a simple list) that maps ancodings to the right de/encode functions by returning a `CodecInfo` object once first matched.
22

3-
!!! note "`codecs` import and the `open` built-in function"
3+
`codext` hooks `codecs`'s functions to insert its own proxy registry between the function calls and the native registry so that new encodings can be added or replace existing ones while using `code[cs|xt].open`. Indeed, as the proxy registry is called first, the first possible match occurs in a custom codec, while if not existing, the native registry is used.
4+
5+
!!! note "The `open` built-in function"
6+
7+
Two behaviors are to be considered when using `codext`:
48

5-
When `codext` is imported, the new encodings are added to its registry but also to the native one. Moreover, hooked functions are bound to the `codext` module but also overwrites the original ones in `codecs`. Consequently:
9+
1. Encodings added from `codext` are only added to the proxy codecs registry of `codext` and are NOT available using `open(...)` (but well using `code[cs|xt].open(...)`.
10+
2. Encodings added from `codecs` are added to the proxy registry AND ALSO to the native registry and are therefore available using `open(...)`.
611

7-
1. Once `codext` has been imported, `codecs` can be imported elsewhere in the program and will have the `add` and hooked functions attached, with the new encodings available.
8-
2. While `codecs.open` will handle the new encodings according to `codext`'s registry first, the native `open` function will rely on the native registry only and therefore handle the encoding search functions of `codext` _after_ these of the native registry has it will rely on non-hooked functions.
12+
This difference allows to keep encodings added from `codext` removable while these added from `codecs` are not. This is the consequence from the fact that there is no unregister function in the native `_codecs` library.
913

1014
-----
1115

@@ -18,16 +22,20 @@ New codecs can be added easily using the new function `add`.
1822
>>> help(codext.add)
1923
Help on function add in module codext.__common__:
2024

21-
add(ename, encode=None, decode=None, pattern=None, text=True)
25+
add(ename, encode=None, decode=None, pattern=None, text=True, add_to_codecs=False)
2226
This adds a new codec to the codecs module setting its encode and/or decode
2327
functions, eventually dynamically naming the encoding with a pattern and
2428
with file handling (if text is True).
2529

26-
:param ename: encoding name
27-
:param encode: encoding function or None
28-
:param decode: decoding function or None
29-
:param pattern: pattern for dynamically naming the encoding
30-
:param text: specify whether the codec is a text encoding
30+
:param ename: encoding name
31+
:param encode: encoding function or None
32+
:param decode: decoding function or None
33+
:param pattern: pattern for dynamically naming the encoding
34+
:param text: specify whether the codec is a text encoding
35+
:param add_to_codecs: also add the search function to the native registry
36+
NB: this will make the codec available in the
37+
built-in open(...) but will make it impossible
38+
to remove the codec later
3139

3240
```
3341

@@ -78,6 +86,61 @@ In this second example, we can see that:
7886

7987
-----
8088

89+
## Remove a custom encoding
90+
91+
New codecs can be removed easily using the new function `remove`, which will only remove every codec matching the given encoding name in the proxy codecs registry and NOT in the native one.
92+
93+
```python
94+
>>> codext.encode("test", "bin")
95+
'01110100011001010111001101110100'
96+
>>> codext.remove("bin")
97+
>>> codext.encode("test", "bin")
98+
99+
Traceback (most recent call last):
100+
File "<pyshell#39>", line 1, in <module>
101+
codext.encode("test", "bin")
102+
File "codext/__common__.py", line 245, in __encode
103+
return __lookup(encoding).encode(obj, errors)[0]
104+
File "codext/__common__.py", line 259, in __lookup
105+
codecs.lookup = __lookup
106+
LookupError: unknown encoding: bin
107+
```
108+
109+
While trying to remove a codec that is in the native registry won't raise a `LookupError`.
110+
111+
```python
112+
>>> codext.remove("utf-8")
113+
>>> codext.encode("test", "utf-8")
114+
b'test'
115+
```
116+
117+
-----
118+
119+
## Remove or restore `codext` encodings
120+
121+
It can be useful while playing with encodings e.g. from Idle to be able to remove or restore `codext`'s encodings. This can be achieved using respectively the new `clear` and `reset` functions.
122+
123+
```python
124+
>>> codext.clear()
125+
>>> codext.encode("test", "bin")
126+
Traceback (most recent call last):
127+
File "<pyshell#4>", line 1, in <module>
128+
codext.encode("test", "bin")
129+
File "/mnt/data/Projects/maint/python-codext/codext/__common__.py", line 245, in __encode
130+
return __lookup(encoding).encode(obj, errors)[0]
131+
File "/mnt/data/Projects/maint/python-codext/codext/__common__.py", line 258, in __lookup
132+
return orig_lookup(encoding)
133+
LookupError: unknown encoding: bin
134+
```
135+
136+
```python
137+
>>> codext.reset()
138+
>>> codext.encode("test", "bin")
139+
'01110100011001010111001101110100'
140+
```
141+
142+
-----
143+
81144
## Hooked `codecs` functions
82145

83146
In order to select the right de/encoding function and avoid any conflict, the native `codecs` library registers search functions (using the `register(search_function)` function), called in order of registration while searching for a codec.

0 commit comments

Comments
 (0)