@@ -311,6 +311,12 @@ pub fn tokenize<'a>(input: &'a str) -> Result<Vec<Token<'a>>, TokenizeError> {
311311 if initial == '(' || initial == '[' || initial == '{' {
312312 parenlev += 1 ;
313313 } else if initial == ')' || initial == ']' || initial == '}' {
314+ if parenlev == 0 {
315+ return Err ( TokenizeError {
316+ msg : "Unbalanced brackets" ,
317+ offset : line_pos,
318+ } ) ;
319+ }
314320 parenlev -= 1 ;
315321 }
316322 result. push ( Token {
@@ -321,18 +327,29 @@ pub fn tokenize<'a>(input: &'a str) -> Result<Vec<Token<'a>>, TokenizeError> {
321327 } ) ;
322328 }
323329 } else {
330+ let char = line[ line_pos..] . chars ( ) . next ( ) . unwrap ( ) ;
331+ let len = char. len_utf8 ( ) ;
332+ let string = & line[ line_pos..line_pos + len] ;
324333 #[ allow( clippy:: range_plus_one) ]
325334 result. push ( Token {
326335 typ : ERRORTOKEN ,
327- string : & line [ line_pos..line_pos + 1 ] ,
328- span : Span :: new ( line_start + line_pos, line_start + line_pos + 1 ) ,
336+ string,
337+ span : Span :: new ( line_start + line_pos, line_start + line_pos + len ) ,
329338 line,
330339 } ) ;
331- line_pos += 1 ;
340+ line_pos += len ;
332341 }
333342 }
334343 }
335344
345+ // Ensure brackets are balanced
346+ if parenlev != 0 {
347+ return Err ( TokenizeError {
348+ msg : "Unbalanced brackets" ,
349+ offset : input. len ( ) ,
350+ } ) ;
351+ }
352+
336353 // We use this zero-length slice as the ending content for remaining tokens.
337354 // This is *just in case* anyone actually cares that the location of the
338355 // pointer makes any kind of sense.
@@ -382,3 +399,40 @@ pub fn tokenize<'a>(input: &'a str) -> Result<Vec<Token<'a>>, TokenizeError> {
382399
383400 Ok ( result)
384401}
402+
403+ #[ cfg( test) ]
404+ mod tests {
405+ use super :: * ;
406+
407+ #[ test]
408+ fn test_balanced_brackets ( ) {
409+ let inputs = [ "[]" , "[[]]" , "()" , "(())" , "{}" , "{{}}" ] ;
410+ for input in & inputs {
411+ assert ! ( tokenize( input) . is_ok( ) ) ;
412+ }
413+ }
414+
415+ #[ test]
416+ fn test_unbalanced_brackets ( ) {
417+ let inputs = [ "[[]" , "[]]" , "(()" , "())" , "{{}" , "{}}" ] ;
418+ for input in & inputs {
419+ assert ! ( tokenize( input) . is_err( ) ) ;
420+ }
421+ }
422+
423+ #[ test]
424+ fn test_unicode_token ( ) {
425+ let uni = "\u{6dd} " ;
426+ let input = format ! ( "[{}]" , uni) ;
427+ let result = tokenize ( & input) ;
428+ assert ! ( result. is_ok( ) ) ;
429+ let tokens = result. unwrap ( ) ;
430+ let token = tokens
431+ . iter ( )
432+ . filter ( |token| token. typ == ERRORTOKEN )
433+ . nth ( 0 )
434+ . unwrap ( ) ;
435+ assert_eq ! ( token. typ, ERRORTOKEN ) ;
436+ assert_eq ! ( token. string, uni) ;
437+ }
438+ }
0 commit comments