Skip to content

Commit 8b134ea

Browse files
TrundleTrundle
committed
Make paren highlighting more accurate.
1 parent 3d412b7 commit 8b134ea

File tree

3 files changed

+109
-78
lines changed

3 files changed

+109
-78
lines changed

‎bpython/cli.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -829,7 +829,7 @@ def print_line(self, s, clr=False, newline=False):
829829

830830
if self.highlighted_paren is not None:
831831
# Clear previous highlighted paren
832-
self.reprint_line(self.highlighted_paren)
832+
self.reprint_line(*self.highlighted_paren)
833833
self.highlighted_paren = None
834834

835835
if OPTS.syntax and (not self.paste_mode or newline):
@@ -937,7 +937,7 @@ def repl(self):
937937
self.prev_block_finished = stdout_position
938938
self.s = ''
939939

940-
def reprint_line(self, lineno, tokens=None):
940+
def reprint_line(self, lineno, tokens):
941941
"""Helper function for paren highlighting: Reprint line at offset
942942
`lineno` in current input buffer."""
943943
if not self.buffer or lineno == len(self.buffer):
@@ -954,8 +954,6 @@ def reprint_line(self, lineno, tokens=None):
954954
return
955955

956956
self.scr.move(real_lineno, 4)
957-
if tokens is None:
958-
tokens = list(PythonLexer().get_tokens(self.buffer[lineno]))
959957
line = format(tokens, BPythonFormatter(OPTS.color_scheme))
960958
for string in line.split('\x04'):
961959
self.echo(string)
@@ -1125,6 +1123,8 @@ def tab(self):
11251123
otherwise attempt to autocomplete to the best match of possible
11261124
choices in the match list."""
11271125

1126+
print >> sys.__stderr__, 'lala:', repr(self.current_string())
1127+
11281128
if self.atbol():
11291129
x_pos = len(self.s) - self.cpos
11301130
num_spaces = x_pos % OPTS.tab_length

‎bpython/gtk_.py‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ def highlight(self):
415415
"""
416416
if OPTS.syntax:
417417
if self.highlighted_paren is not None:
418-
self.reprint_line(self.highlighted_paren)
418+
self.reprint_line(*self.highlighted_paren)
419419
self.highlighted_paren = None
420420

421421
offset = self.get_cursor_iter().get_offset()
@@ -434,12 +434,6 @@ def insert_highlighted(self, iter_, string):
434434
def insert_highlighted_tokens(self, iter_, tokens):
435435
offset = iter_.get_offset()
436436
buffer = self.text_buffer
437-
# Unfortunately, Pygments adds a trailing newline and strings with
438-
# no size, so strip them
439-
tokens = list(tokens)
440-
while not tokens[-1][1]:
441-
tokens.pop()
442-
tokens[-1] = (tokens[-1][0], tokens[-1][1].rstrip('\n'))
443437
for (token, value) in tokens:
444438
while token not in theme_map:
445439
token = token.parent
@@ -489,7 +483,8 @@ def on_buf_insert_text(self, buffer, iter_, text, length):
489483
self.complete()
490484

491485
def on_buf_mark_set(self, buffer, iter_, textmark):
492-
if textmark.get_name() == 'insert':
486+
name = textmark.get_name()
487+
if name == 'insert':
493488
line_start = self.get_line_start_iter()
494489
if line_start.compare(iter_) > 0:
495490
# Don't set cursor before the start of line
@@ -538,7 +533,7 @@ def push_line(self):
538533
self.highlight()
539534
return self.push(line + '\n')
540535

541-
def reprint_line(self, lineno, tokens=None):
536+
def reprint_line(self, lineno, tokens):
542537
"""
543538
Helper function for paren highlighting: Reprint line at offset
544539
`lineno` in current input buffer.
@@ -552,8 +547,6 @@ def reprint_line(self, lineno, tokens=None):
552547
end.forward_to_line_end()
553548
with self.editing:
554549
self.text_buffer.delete(start, end)
555-
if tokens is None:
556-
tokens = PythonLexer().get_tokens(self.buffer[lineno])
557550
start = self.text_buffer.get_iter_at_mark(mark)
558551
self.insert_highlighted_tokens(start, tokens)
559552

‎bpython/repl.py‎

Lines changed: 101 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import textwrap
3434
import traceback
3535
from glob import glob
36+
from itertools import takewhile
3637
from locale import getpreferredencoding
3738
from urlparse import urljoin
3839
from xmlrpclib import ServerProxy, Error as XMLRPCError
@@ -274,7 +275,6 @@ def __init__(self, interp, idle=None):
274275
self.matches_iter = MatchesIterator()
275276
self.argspec = None
276277
self.current_func = None
277-
self.inside_string = False
278278
self.highlighted_paren = None
279279
self.list_win_visible = False
280280
self._C = {}
@@ -334,23 +334,34 @@ def _callable_postfix(self, value, word):
334334
word += '('
335335
return word
336336

337-
def current_string(self):
338-
"""Return the current string.
339-
Note: This method will not really work for multiline strings."""
340-
line = self.current_line()
341-
inside_string = next_token_inside_string(line, self.inside_string)
342-
if inside_string:
343-
string = list()
344-
next_char = ''
345-
for (char, next_char) in zip(reversed(line),
346-
reversed(line[:-1])):
347-
if char == inside_string and next_char != '\\':
348-
return ''.join(reversed(string))
349-
string.append(char)
337+
def current_string(self, concatenate=False):
338+
"""Return the current string."""
339+
tokens = self.tokenize(self.current_line())
340+
string_tokens = list(takewhile(token_is_any_of([Token.String,
341+
Token.Text]),
342+
reversed(tokens)))
343+
if not string_tokens:
344+
return ''
345+
opening = string_tokens.pop()[1]
346+
string = list()
347+
for (token, value) in reversed(string_tokens):
348+
if token is Token.Text:
349+
continue
350+
elif opening is None:
351+
opening = value
352+
elif token is Token.String.Doc:
353+
string.append(value[3:-3])
354+
opening = None
355+
elif value == opening:
356+
opening = None
357+
if not concatenate:
358+
string = list()
350359
else:
351-
if next_char == inside_string:
352-
return ''.join(reversed(string))
353-
return ''
360+
string.append(value)
361+
362+
if opening is None:
363+
return ''
364+
return ''.join(string)
354365

355366
def get_object(self, name):
356367
if name in self.interp.locals:
@@ -602,8 +613,6 @@ def push(self, s, insert_into_history=True):
602613
if insert_into_history:
603614
self.rl_history.append(s)
604615

605-
self.inside_string = next_token_inside_string(s, self.inside_string)
606-
607616
try:
608617
more = self.interp.runsource('\n'.join(self.buffer))
609618
except SystemExit:
@@ -707,36 +716,45 @@ def close(self):
707716

708717
def tokenize(self, s, newline=False):
709718
"""Tokenize a line of code."""
710-
if self.inside_string:
711-
# A string started in another line is continued in this
712-
# line
713-
tokens = PythonLexer().get_tokens(self.inside_string + s)
714-
token, value = tokens.next()
715-
if token is Token.String.Doc:
716-
tokens = [(Token.String, value[3:])] + list(tokens)
717-
else:
718-
tokens = list(PythonLexer().get_tokens(s))
719719

720720
source = '\n'.join(self.buffer + [s])
721721
cursor = len(source) - self.cpos
722722
if self.cpos:
723723
cursor += 1
724724
stack = list()
725725
all_tokens = list(PythonLexer().get_tokens(source))
726-
i = line = 0
727-
pos = 0
726+
# Unfortunately, Pygments adds a trailing newline and strings with
727+
# no size, so strip them
728+
while not all_tokens[-1][1]:
729+
all_tokens.pop()
730+
all_tokens[-1] = (all_tokens[-1][0], all_tokens[-1][1].rstrip('\n'))
731+
line = pos = 0
728732
parens = dict(zip('{([', '})]'))
729-
for (token, value) in all_tokens:
733+
line_tokens = list()
734+
saved_tokens = list()
735+
search_for_paren = True
736+
for (token, value) in split_lines(all_tokens):
730737
pos += len(value)
738+
if token is Token.Text and value == '\n':
739+
line += 1
740+
# Remove trailing newline
741+
line_tokens = list()
742+
saved_tokens = list()
743+
continue
744+
line_tokens.append((token, value))
745+
saved_tokens.append((token, value))
746+
if not search_for_paren:
747+
continue
731748
under_cursor = (pos == cursor)
732749
if token is Token.Punctuation:
733750
if value in parens:
734751
if under_cursor:
735-
tokens[i] = (Parenthesis.UnderCursor, value)
752+
line_tokens[-1] = (Parenthesis.UnderCursor, value)
736753
# Push marker on the stack
737754
stack.append((Parenthesis, value))
738755
else:
739-
stack.append((line, i, value))
756+
stack.append((line, len(line_tokens) - 1,
757+
line_tokens, value))
740758
elif value in parens.itervalues():
741759
saved_stack = list(stack)
742760
try:
@@ -747,58 +765,45 @@ def tokenize(self, s, newline=False):
747765
except IndexError:
748766
# SyntaxError.. more closed parentheses than
749767
# opened or a wrong closing paren
768+
opening = None
750769
if not saved_stack:
751-
break
770+
search_for_paren = False
752771
else:
753-
opening = None
754772
stack = saved_stack
755773
if opening and opening[0] is Parenthesis:
756774
# Marker found
757-
tokens[i] = (Parenthesis, value)
758-
break
775+
line_tokens[-1] = (Parenthesis, value)
776+
search_for_paren = False
759777
elif opening and under_cursor and not newline:
760778
if self.cpos:
761-
tokens[i] = (Parenthesis.UnderCursor, value)
779+
line_tokens[-1] = (Parenthesis.UnderCursor, value)
762780
else:
763781
# The cursor is at the end of line and next to
764782
# the paren, so it doesn't reverse the paren.
765783
# Therefore, we insert the Parenthesis token
766784
# here instead of the Parenthesis.UnderCursor
767785
# token.
768-
if i < len(tokens):
769-
# XXX This is a bug in our index
770-
# calculation (happens with multiline
771-
# strings)
772-
tokens[i] = (Parenthesis, value)
773-
(lineno, i, opening) = opening
786+
line_tokens[-1] = (Parenthesis, value)
787+
(lineno, i, tokens, opening) = opening
774788
if lineno == len(self.buffer):
775-
self.highlighted_paren = lineno
776-
tokens[i] = (Parenthesis, opening)
789+
self.highlighted_paren = (lineno, saved_tokens)
790+
line_tokens[i] = (Parenthesis, opening)
777791
else:
778-
line = self.buffer[lineno]
779-
self.highlighted_paren = lineno
792+
self.highlighted_paren = (lineno, list(tokens))
780793
# We need to redraw a line
781-
line_tokens = list(PythonLexer().get_tokens(line))
782-
line_tokens[i] = (Parenthesis, opening)
783-
self.reprint_line(lineno, line_tokens)
784-
break
794+
tokens[i] = (Parenthesis, opening)
795+
self.reprint_line(lineno, tokens)
796+
search_for_paren = False
785797
elif under_cursor:
786-
break
787-
elif token is Token.Text and value == '\n':
788-
line += 1
789-
if line == len(self.buffer):
790-
i = -1
791-
elif under_cursor:
792-
break
793-
i += 1
794-
return tokens
798+
search_for_paren = False
799+
if line != len(self.buffer):
800+
return list()
801+
return line_tokens
795802

796803
def clear_current_line(self):
797804
"""This is used as the exception callback for the Interpreter instance.
798805
It prevents autoindentation from occuring after a traceback."""
799806

800-
self.inside_string = False
801-
802807

803808
def next_indentantion(line):
804809
"""Given a code line, return the indentation of the next line."""
@@ -821,3 +826,36 @@ def next_token_inside_string(s, inside_string):
821826
elif value == inside_string:
822827
inside_string = False
823828
return inside_string
829+
830+
831+
def split_lines(tokens):
832+
for (token, value) in tokens:
833+
if not value:
834+
continue
835+
while value:
836+
head, newline, value = value.partition('\n')
837+
yield (token, head)
838+
if newline:
839+
yield (Token.Text, newline)
840+
841+
def token_is(token_type):
842+
"""Return a callable object that returns whether a token is of the
843+
given type `token_type`."""
844+
def token_is_type(token):
845+
"""Return whether a token is of a certain type or not."""
846+
token = token[0]
847+
while token is not token_type and token.parent:
848+
token = token.parent
849+
return token is token_type
850+
851+
return token_is_type
852+
853+
def token_is_any_of(token_types):
854+
"""Return a callable object that returns whether a token is any of the
855+
given types `token_types`."""
856+
is_token_types = map(token_is, token_types)
857+
858+
def token_is_any_of(token):
859+
return any(check(token) for check in is_token_types)
860+
861+
return token_is_any_of

0 commit comments

Comments
 (0)