Skip to content

Commit 97ed070

Browse files
author
derwentx
committed
more sane encoding
1 parent da3dbd9 commit 97ed070

File tree

5 files changed

+120
-95
lines changed

5 files changed

+120
-95
lines changed

‎tests.py‎

Lines changed: 52 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
""" API Tests """
2+
from __future__ import unicode_literals
3+
24
import functools
35
import logging
46
import pdb
@@ -14,23 +16,14 @@
1416
import six
1517
import wordpress
1618
from httmock import HTTMock, all_requests, urlmatch
17-
from six import text_type, u
19+
from six import text_type
20+
from six.moves.urllib.parse import parse_qsl, urlparse
1821
from wordpress import __default_api__, __default_api_version__, auth
1922
from wordpress.api import API
2023
from wordpress.auth import Auth, OAuth
2124
from wordpress.helpers import SeqUtils, StrUtils, UrlUtils
2225
from wordpress.transport import API_Requests_Wrapper
2326

24-
try:
25-
from urllib.parse import (
26-
urlencode, quote, unquote, parse_qs, parse_qsl, urlparse, urlunparse
27-
)
28-
from urllib.parse import ParseResult as URLParseResult
29-
except ImportError:
30-
from urllib import urlencode, quote, unquote
31-
from urlparse import parse_qs, parse_qsl, urlparse, urlunparse
32-
from urlparse import ParseResult as URLParseResult
33-
3427

3528
def debug_on(*exceptions):
3629
if not exceptions:
@@ -55,6 +48,7 @@ def wrapper(*args, **kwargs):
5548

5649
CURRENT_TIMESTAMP = int(time())
5750
SHITTY_NONCE = ""
51+
DEFAULT_ENCODING = sys.getdefaultencoding()
5852

5953

6054
class WordpressTestCase(unittest.TestCase):
@@ -1007,47 +1001,71 @@ def test_APIPutWithSimpleQuery(self):
10071001

10081002
wcapi.put('products/%s' % (product_id), {"name": original_title})
10091003

1004+
@unittest.skipIf(six.PY2, "non-utf8 bytes not supported in python2")
1005+
def test_APIPostWithBytesQuery(self):
1006+
wcapi = API(**self.api_params)
1007+
nonce = b"%f\xff" % random.random()
1008+
1009+
data = {
1010+
"name": nonce,
1011+
"type": "simple",
1012+
}
1013+
1014+
response = wcapi.post('products', data)
1015+
response_obj = response.json()
1016+
product_id = response_obj.get('id')
1017+
1018+
expected = StrUtils.to_text(nonce, encoding='ascii', errors='replace')
1019+
1020+
self.assertEqual(
1021+
response_obj.get('name'),
1022+
expected,
1023+
)
1024+
wcapi.delete('products/%s' % product_id)
1025+
1026+
@unittest.skipIf(six.PY2, "non-utf8 bytes not supported in python2")
10101027
def test_APIPostWithLatin1Query(self):
10111028
wcapi = API(**self.api_params)
1012-
nonce = u"%f\u00ae" % random.random()
1029+
nonce = "%f\u00ae" % random.random()
10131030

10141031
data = {
10151032
"name": nonce.encode('latin-1'),
10161033
"type": "simple",
10171034
}
10181035

1019-
if six.PY2:
1020-
response = wcapi.post('products', data)
1021-
response_obj = response.json()
1022-
product_id = response_obj.get('id')
1023-
self.assertEqual(response_obj.get('name'), nonce)
1024-
wcapi.delete('products/%s' % product_id)
1025-
return
1026-
with self.assertRaises(TypeError):
1027-
response = wcapi.post('products', data)
1036+
response = wcapi.post('products', data)
1037+
response_obj = response.json()
1038+
product_id = response_obj.get('id')
1039+
1040+
expected = StrUtils.to_text(
1041+
StrUtils.to_binary(nonce, encoding='latin-1'),
1042+
encoding='ascii', errors='replace'
1043+
)
1044+
1045+
self.assertEqual(
1046+
response_obj.get('name'),
1047+
expected
1048+
)
1049+
wcapi.delete('products/%s' % product_id)
10281050

10291051
def test_APIPostWithUTF8Query(self):
10301052
wcapi = API(**self.api_params)
1031-
nonce = u"%f\u00ae" % random.random()
1053+
nonce = "%f\u00ae" % random.random()
10321054

10331055
data = {
10341056
"name": nonce.encode('utf8'),
10351057
"type": "simple",
10361058
}
10371059

1038-
if six.PY2:
1039-
response = wcapi.post('products', data)
1040-
response_obj = response.json()
1041-
product_id = response_obj.get('id')
1042-
self.assertEqual(response_obj.get('name'), nonce)
1043-
wcapi.delete('products/%s' % product_id)
1044-
return
1045-
with self.assertRaises(TypeError):
1046-
response = wcapi.post('products', data)
1060+
response = wcapi.post('products', data)
1061+
response_obj = response.json()
1062+
product_id = response_obj.get('id')
1063+
self.assertEqual(response_obj.get('name'), nonce)
1064+
wcapi.delete('products/%s' % product_id)
10471065

10481066
def test_APIPostWithUnicodeQuery(self):
10491067
wcapi = API(**self.api_params)
1050-
nonce = u"%f\u00ae" % random.random()
1068+
nonce = "%f\u00ae" % random.random()
10511069

10521070
data = {
10531071
"name": nonce,
@@ -1100,7 +1118,7 @@ def test_APIGetWithSimpleQuery(self):
11001118
self.assertEqual(len(response_obj), 2)
11011119

11021120
def test_APIPostData(self):
1103-
nonce = u"%f\u00ae" % random.random()
1121+
nonce = "%f\u00ae" % random.random()
11041122

11051123
content = "api test post"
11061124

@@ -1120,7 +1138,7 @@ def test_APIPostBadData(self):
11201138
"""
11211139
No excerpt so should fail to be created.
11221140
"""
1123-
nonce = u"%f\u00ae" % random.random()
1141+
nonce = "%f\u00ae" % random.random()
11241142

11251143
data = {
11261144
'a': nonce

‎wordpress/api.py‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,18 @@
44
Wordpress API Class
55
"""
66

7-
__title__ = "wordpress-api"
7+
from __future__ import unicode_literals
88

99
# from requests import request
1010
import logging
11-
from json import dumps as jsonencode
1211

1312
from six import text_type
1413
from wordpress.auth import BasicAuth, NoAuth, OAuth, OAuth_3Leg
1514
from wordpress.helpers import StrUtils, UrlUtils
1615
from wordpress.transport import API_Requests_Wrapper
1716

17+
__title__ = "wordpress-api"
18+
1819

1920
class API(object):
2021
""" API Class """
@@ -113,7 +114,7 @@ def request_post_mortem(self, response=None):
113114
isinstance(response_json, dict)
114115
and ('code' in response_json or 'message' in response_json)
115116
):
116-
reason = u" - ".join([
117+
reason = " - ".join([
117118
text_type(response_json.get(key))
118119
for key in ['code', 'message', 'data']
119120
if key in response_json
@@ -180,15 +181,15 @@ def request_post_mortem(self, response=None):
180181
remedy = "try changing url to %s" % header_url
181182

182183
msg = (
183-
u"API call to %s returned \nCODE: "
184+
"API call to %s returned \nCODE: "
184185
"%s\nRESPONSE:%s \nHEADERS: %s\nREQ_BODY:%s"
185-
) % (
186+
) % tuple(map(StrUtils.to_text, [
186187
request_url,
187-
text_type(response.status_code),
188+
response.status_code,
188189
UrlUtils.beautify_response(response),
189-
text_type(response_headers),
190-
StrUtils.to_binary(request_body)[:1000]
191-
)
190+
response_headers,
191+
request_body[:1000]
192+
]))
192193
if reason:
193194
msg += "\nBecause of %s" % StrUtils.to_binary(reason)
194195
if remedy:
@@ -206,9 +207,10 @@ def __request(self, method, endpoint, data, **kwargs):
206207
'content-type', 'application/json')
207208

208209
if data is not None and content_type.startswith('application/json'):
209-
data = jsonencode(data, ensure_ascii=False)
210+
data = StrUtils.jsonencode(data, ensure_ascii=False)
211+
210212
# enforce utf-8 encoded binary
211-
data = StrUtils.to_binary(data, encoding='utf8')
213+
data = StrUtils.to_binary(data)
212214

213215
response = self.requester.request(
214216
method=method,

‎wordpress/auth.py‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,40 +6,26 @@
66

77
__title__ = "wordpress-auth"
88

9-
# from base64 import b64encode
109
import binascii
1110
import json
1211
import logging
1312
import os
13+
from collections import OrderedDict
1414
from hashlib import sha1, sha256
1515
from hmac import new as HMAC
1616
from pprint import pformat
1717
from random import randint
1818
from time import time
1919

20-
# import webbrowser
2120
import requests
2221
from requests.auth import HTTPBasicAuth
2322

2423
from bs4 import BeautifulSoup
24+
from six.moves.urllib.parse import parse_qs, parse_qsl, quote, urlparse
2525
from wordpress import __version__
2626

2727
from .helpers import StrUtils, UrlUtils
2828

29-
try:
30-
from urllib.parse import (urlencode, quote, unquote, parse_qs, parse_qsl,
31-
urlparse, urlunparse)
32-
from urllib.parse import ParseResult as URLParseResult
33-
except ImportError:
34-
from urllib import urlencode, quote, unquote
35-
from urlparse import parse_qs, parse_qsl, urlparse, urlunparse
36-
from urlparse import ParseResult as URLParseResult
37-
38-
try:
39-
from collections import OrderedDict
40-
except ImportError:
41-
from ordereddict import OrderedDict
42-
4329

4430
class Auth(object):
4531
""" Boilerplate for handling authentication stuff. """
@@ -492,13 +478,9 @@ def get_form_info(self, response, form_id):
492478
% (form_soup.prettify()).encode('ascii', errors='backslashreplace')
493479

494480
form_data = OrderedDict()
495-
for input_soup in form_soup.select('input') + form_soup.select('button'):
496-
# print "input, class:%5s, id=%5s, name=%5s, value=%s" % (
497-
# input_soup.get('class'),
498-
# input_soup.get('id'),
499-
# input_soup.get('name'),
500-
# input_soup.get('value')
501-
# )
481+
for input_soup in (
482+
form_soup.select('input') + form_soup.select('button')
483+
):
502484
name = input_soup.get('name')
503485
if not name:
504486
continue

‎wordpress/helpers.py‎

Lines changed: 50 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,24 @@
11
# -*- coding: utf-8 -*-
22

33
"""
4-
Wordpress Hellpers Class
4+
Wordpress Hellper Class
55
"""
66

77
__title__ = "wordpress-requests"
88

9+
import json
910
import posixpath
1011
import re
12+
import sys
1113
from collections import OrderedDict
1214

1315
from bs4 import BeautifulSoup
14-
from six import binary_type, text_type
16+
from six import (PY2, PY3, binary_type, iterbytes, string_types, text_type,
17+
unichr)
1518
from six.moves import reduce
16-
17-
try:
18-
from urllib.parse import (urlencode, quote, unquote, parse_qs, parse_qsl,
19-
urlparse, urlunparse)
20-
from urllib.parse import ParseResult as URLParseResult
21-
except ImportError:
22-
from urllib import urlencode, quote, unquote
23-
from urlparse import parse_qs, parse_qsl, urlparse, urlunparse
24-
from urlparse import ParseResult as URLParseResult
19+
from six.moves.urllib.parse import ParseResult as URLParseResult
20+
from six.moves.urllib.parse import (parse_qs, parse_qsl, quote, urlencode,
21+
urlparse, urlunparse)
2522

2623

2724
class StrUtils(object):
@@ -46,19 +43,54 @@ def eviscerate(cls, *args, **kwargs):
4643
return cls.remove_tail(*args, **kwargs)
4744

4845
@classmethod
49-
def to_binary(cls, string, encoding='utf8', errors='backslashreplace'):
46+
def to_text(cls, string, encoding='utf-8', errors='replace'):
47+
if isinstance(string, text_type):
48+
return string
5049
if isinstance(string, binary_type):
5150
try:
52-
string = string.decode('utf8')
53-
except UnicodeDecodeError:
54-
string = string.decode('latin-1')
51+
return string.decode(encoding, errors=errors)
52+
except TypeError:
53+
return ''.join([
54+
unichr(c) for c in iterbytes(string)
55+
])
56+
return text_type(string)
57+
58+
@classmethod
59+
def to_binary(cls, string, encoding='utf8', errors='backslashreplace'):
60+
if isinstance(string, binary_type):
61+
return string
5562
if not isinstance(string, text_type):
5663
string = text_type(string)
57-
return string.encode(encoding, errors=errors)
64+
return string.encode(encoding, errors)
5865

5966
@classmethod
60-
def to_binary_ascii(cls, string):
61-
return cls.to_binary(string, 'ascii')
67+
def jsonencode(cls, data, **kwargs):
68+
# kwargs['cls'] = BytesJsonEncoder
69+
# if PY2:
70+
# kwargs['encoding'] = 'utf8'
71+
if PY2:
72+
for encoding in [
73+
kwargs.get('encoding', 'utf8'),
74+
sys.getdefaultencoding(),
75+
'utf8',
76+
]:
77+
try:
78+
kwargs['encoding'] = encoding
79+
return json.dumps(data, **kwargs)
80+
except UnicodeDecodeError:
81+
pass
82+
kwargs.pop('encoding', None)
83+
kwargs['cls'] = BytesJsonEncoder
84+
return json.dumps(data, **kwargs)
85+
86+
87+
class BytesJsonEncoder(json.JSONEncoder):
88+
def default(self, obj):
89+
90+
if isinstance(obj, binary_type):
91+
return StrUtils.to_text(obj, errors='replace')
92+
# Let the base class default method raise the TypeError
93+
return json.JSONEncoder.default(self, obj)
6294

6395

6496
class SeqUtils(object):

‎wordpress/transport.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,6 @@
1414
from wordpress import __default_api__, __default_api_version__, __version__
1515
from wordpress.helpers import SeqUtils, StrUtils, UrlUtils
1616

17-
try:
18-
from urllib.parse import (urlencode, quote, unquote, parse_qsl, urlparse,
19-
urlunparse)
20-
from urllib.parse import ParseResult as URLParseResult
21-
except ImportError:
22-
from urllib import urlencode, quote, unquote
23-
from urlparse import parse_qsl, urlparse, urlunparse
24-
from urlparse import ParseResult as URLParseResult
25-
2617

2718
class API_Requests_Wrapper(object):
2819
""" provides a wrapper for making requests that handles session info """

0 commit comments

Comments
 (0)