Skip to content

Commit f00c49d

Browse files
committed
Issue python#16182: Fix readline begidx, endidx, and use locale encoding
Based on patch by Serhiy Storchaka.
1 parent 19e21e4 commit f00c49d

3 files changed

Lines changed: 235 additions & 42 deletions

File tree

‎Lib/test/test_readline.py‎

Lines changed: 137 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,30 @@
11
"""
22
Very minimal unittests for parts of the readline module.
33
"""
4+
from contextlib import ExitStack
5+
from errno import EIO
46
import os
7+
import selectors
8+
import subprocess
9+
import sys
510
import tempfile
611
import unittest
7-
from test.support import import_module, unlink
12+
from test.support import import_module, unlink, TESTFN
813
from test.support.script_helper import assert_python_ok
914

1015
# Skip tests if there is no readline module
1116
readline = import_module('readline')
1217

18+
@unittest.skipUnless(hasattr(readline, "clear_history"),
19+
"The history update test cannot be run because the "
20+
"clear_history method is not available.")
1321
class TestHistoryManipulation (unittest.TestCase):
1422
"""
1523
These tests were added to check that the libedit emulation on OSX and the
1624
"real" readline have the same interface for history manipulation. That's
1725
why the tests cover only a small subset of the interface.
1826
"""
1927

20-
@unittest.skipUnless(hasattr(readline, "clear_history"),
21-
"The history update test cannot be run because the "
22-
"clear_history method is not available.")
2328
def testHistoryUpdates(self):
2429
readline.clear_history()
2530

@@ -82,6 +87,21 @@ def test_write_read_append(self):
8287
# write_history_file can create the target
8388
readline.write_history_file(hfilename)
8489

90+
def test_nonascii_history(self):
91+
readline.clear_history()
92+
try:
93+
readline.add_history("entrée 1")
94+
except UnicodeEncodeError as err:
95+
self.skipTest("Locale cannot encode test data: " + format(err))
96+
readline.add_history("entrée 2")
97+
readline.replace_history_item(1, "entrée 22")
98+
readline.write_history_file(TESTFN)
99+
self.addCleanup(os.remove, TESTFN)
100+
readline.clear_history()
101+
readline.read_history_file(TESTFN)
102+
self.assertEqual(readline.get_history_item(1), "entrée 1")
103+
self.assertEqual(readline.get_history_item(2), "entrée 22")
104+
85105

86106
class TestReadline(unittest.TestCase):
87107

@@ -96,6 +116,119 @@ def test_init(self):
96116
TERM='xterm-256color')
97117
self.assertEqual(stdout, b'')
98118

119+
def test_nonascii(self):
120+
try:
121+
readline.add_history("\xEB\xEF")
122+
except UnicodeEncodeError as err:
123+
self.skipTest("Locale cannot encode test data: " + format(err))
124+
125+
script = r"""import readline
126+
127+
if readline.__doc__ and "libedit" in readline.__doc__:
128+
readline.parse_and_bind(r'bind ^B ed-prev-char')
129+
readline.parse_and_bind(r'bind "\t" rl_complete')
130+
readline.parse_and_bind('bind -s ^A "|t\xEB[after]"')
131+
else:
132+
readline.parse_and_bind(r'Control-b: backward-char')
133+
readline.parse_and_bind(r'"\t": complete')
134+
readline.parse_and_bind(r'set disable-completion off')
135+
readline.parse_and_bind(r'set show-all-if-ambiguous off')
136+
readline.parse_and_bind(r'set show-all-if-unmodified off')
137+
readline.parse_and_bind('Control-a: "|t\xEB[after]"')
138+
139+
def pre_input_hook():
140+
readline.insert_text("[\xEFnserted]")
141+
readline.redisplay()
142+
readline.set_pre_input_hook(pre_input_hook)
143+
144+
def completer(text, state):
145+
if text == "t\xEB":
146+
if state == 0:
147+
print("text", ascii(text))
148+
print("line", ascii(readline.get_line_buffer()))
149+
print("indexes", readline.get_begidx(), readline.get_endidx())
150+
return "t\xEBnt"
151+
if state == 1:
152+
return "t\xEBxt"
153+
if text == "t\xEBx" and state == 0:
154+
return "t\xEBxt"
155+
return None
156+
readline.set_completer(completer)
157+
158+
def display(substitution, matches, longest_match_length):
159+
print("substitution", ascii(substitution))
160+
print("matches", ascii(matches))
161+
readline.set_completion_display_matches_hook(display)
162+
163+
print("result", ascii(input()))
164+
print("history", ascii(readline.get_history_item(1)))
165+
"""
166+
167+
input = b"\x01" # Ctrl-A, expands to "|t\xEB[after]"
168+
input += b"\x02" * len("[after]") # Move cursor back
169+
input += b"\t\t" # Display possible completions
170+
input += b"x\t" # Complete "t\xEBx" -> "t\xEBxt"
171+
input += b"\r"
172+
output = run_pty(script, input)
173+
self.assertIn(b"text 't\\xeb'\r\n", output)
174+
self.assertIn(b"line '[\\xefnserted]|t\\xeb[after]'\r\n", output)
175+
self.assertIn(b"indexes 11 13\r\n", output)
176+
self.assertIn(b"substitution 't\\xeb'\r\n", output)
177+
self.assertIn(b"matches ['t\\xebnt', 't\\xebxt']\r\n", output)
178+
expected = br"'[\xefnserted]|t\xebxt[after]'"
179+
self.assertIn(b"result " + expected + b"\r\n", output)
180+
self.assertIn(b"history " + expected + b"\r\n", output)
181+
182+
183+
def run_pty(script, input=b"dummy input\r"):
184+
pty = import_module('pty')
185+
output = bytearray()
186+
[master, slave] = pty.openpty()
187+
args = (sys.executable, '-c', script)
188+
proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave)
189+
os.close(slave)
190+
with ExitStack() as cleanup:
191+
cleanup.enter_context(proc)
192+
def terminate(proc):
193+
try:
194+
proc.terminate()
195+
except ProcessLookupError:
196+
# Workaround for Open/Net BSD bug (Issue 16762)
197+
pass
198+
cleanup.callback(terminate, proc)
199+
cleanup.callback(os.close, master)
200+
# Avoid using DefaultSelector and PollSelector. Kqueue() does not
201+
# work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open
202+
# BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4
203+
# either (Issue 20472). Hopefully the file descriptor is low enough
204+
# to use with select().
205+
sel = cleanup.enter_context(selectors.SelectSelector())
206+
sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE)
207+
os.set_blocking(master, False)
208+
while True:
209+
for [_, events] in sel.select():
210+
if events & selectors.EVENT_READ:
211+
try:
212+
chunk = os.read(master, 0x10000)
213+
except OSError as err:
214+
# Linux raises EIO when slave is closed (Issue 5380)
215+
if err.errno != EIO:
216+
raise
217+
chunk = b""
218+
if not chunk:
219+
return output
220+
output.extend(chunk)
221+
if events & selectors.EVENT_WRITE:
222+
try:
223+
input = input[os.write(master, input):]
224+
except OSError as err:
225+
# Apparently EIO means the slave was closed
226+
if err.errno != EIO:
227+
raise
228+
input = b"" # Stop writing
229+
if not input:
230+
sel.modify(master, selectors.EVENT_READ)
231+
99232

100233
if __name__ == "__main__":
101234
unittest.main()

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ Core and Builtins
1313
Library
1414
-------
1515

16+
- Issue #16182: Fix various functions in the "readline" module to use the
17+
locale encoding, and fix get_begidx() and get_endidx() to return code point
18+
indexes.
19+
1620

1721
What's New in Python 3.5.2 final?
1822
=================================

0 commit comments

Comments
 (0)