Skip to content

Commit a8b24da

Browse files
author
Josh Hoyt
committed
[project @ Make sure that everything has the expected encoding when doing kvform generation and parsing]
Previously, we were not decoding the pairs when parsing KVForm.
1 parent bfce988 commit a8b24da

2 files changed

Lines changed: 29 additions & 8 deletions

File tree

‎openid/kvform.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ def err(msg):
2323

2424
lines = []
2525
for k, v in seq:
26-
if isinstance(k, types.UnicodeType):
27-
k = k.encode('UTF8')
28-
elif not isinstance(k, types.StringType):
26+
if isinstance(k, types.StringType):
27+
k = k.decode('UTF8')
28+
elif not isinstance(k, types.UnicodeType):
2929
err('Converting key to string: %r' % k)
3030
k = str(k)
3131

@@ -40,9 +40,9 @@ def err(msg):
4040
if k.strip() != k:
4141
err('Key has whitespace at beginning or end: %r' % k)
4242

43-
if isinstance(v, types.UnicodeType):
44-
v = v.encode('UTF8')
45-
elif not isinstance(v, types.StringType):
43+
if isinstance(v, types.StringType):
44+
v = v.decode('UTF8')
45+
elif not isinstance(v, types.UnicodeType):
4646
err('Converting value to string: %r' % v)
4747
v = str(v)
4848

@@ -55,7 +55,7 @@ def err(msg):
5555

5656
lines.append(k + ':' + v + '\n')
5757

58-
return ''.join(lines)
58+
return ''.join(lines).encode('UTF8')
5959

6060
def kvToSeq(data, strict=False):
6161
"""
@@ -105,7 +105,7 @@ def err(msg):
105105
'whitespace in value %r')
106106
err(fmt % (line_num, v))
107107

108-
pairs.append((k_s, v_s))
108+
pairs.append((k_s.decode('UTF8'), v_s.decode('UTF8')))
109109
else:
110110
err('Line %d does not contain a colon' % line_num)
111111

‎openid/test/kvform.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,18 @@ def cleanSeq(self, seq):
5757
and end of each value of each pair"""
5858
clean = []
5959
for k, v in self.seq:
60+
if type(k) is str:
61+
k = k.decode('utf8')
62+
if type(v) is str:
63+
v = v.decode('utf8')
6064
clean.append((k.strip(), v.strip()))
6165
return clean
6266

6367
def runTest(self):
6468
# seq serializes to expected kvform
6569
actual = kvform.seqToKV(self.seq)
6670
self.failUnlessEqual(self.kvform, actual)
71+
self.failUnless(type(actual) is str)
6772

6873
# Parse back to sequence. Expected to be unchanged, except
6974
# stripping whitespace from start and end of values
@@ -108,12 +113,28 @@ def runTest(self):
108113

109114
kvseq_cases = [
110115
([], '', 0),
116+
117+
# Make sure that we handle non-ascii characters (also wider than 8 bits)
118+
([(u'\u03bbx', u'x')], '\xce\xbbx:x\n', 0),
119+
120+
# If it's a UTF-8 str, make sure that it's equivalent to the same
121+
# string, decoded.
122+
([('\xce\xbbx', 'x')], '\xce\xbbx:x\n', 0),
123+
111124
([('openid', 'useful'), ('a', 'b')], 'openid:useful\na:b\n', 0),
125+
126+
# Warnings about leading whitespace
112127
([(' openid', 'useful'), ('a', 'b')], ' openid:useful\na:b\n', 2),
128+
129+
# Warnings about leading and trailing whitespace
113130
([(' openid ', ' useful '),
114131
(' a ', ' b ')], ' openid : useful \n a : b \n', 8),
132+
133+
# warnings about leading and trailing whitespace, but not about
134+
# internal whitespace.
115135
([(' open id ', ' use ful '),
116136
(' a ', ' b ')], ' open id : use ful \n a : b \n', 8),
137+
117138
([(u'foo', 'bar')], 'foo:bar\n', 0),
118139
]
119140

0 commit comments

Comments
 (0)