@@ -177,7 +177,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
177177 _PyTok_Span tail = {tok->inp, tok->source.base_offset + tok->source.len};
178178 _PyTok_Off newline = _PyTok_SourceFindByte(&tok->source, tail, '\n');
179179 _PyTok_Span line = {tail.start, newline >= 0 ? newline + 1 : tail.end};
180- chunk->data = (char *)_PyTok_SourceSpanView(&tok->source, line, &chunk->len);
180+ chunk->data = (char *)_PyTok_SourceSpanView(
181+ &tok->source, line, &chunk->len);
181182 chunk->ownership = _PYTOK_CHUNK_BORROWED;
182183 chunk->implicit_newline = line.end == tail.end &&
183184 tok->reader->prepared_final_newline_is_implicit;
@@ -607,7 +608,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
607608 _PyTok_ReaderKind kind = reader->kind;
608609 int prepared = kind == _PYTOK_READER_PREPARED;
609610 int streaming = reader_is_streaming(kind);
610- int reset_buffer = !prepared && tok->start < 0 &&
611+ int reset_buffer = tok->start < 0 &&
611612 _PyLexer_CurrentFTString(tok) == NULL;
612613
613614 _PyTok_Chunk chunk;
@@ -660,12 +661,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
660661 tok->cur = source_start;
661662 tok->buf_offset = source_start;
662663 tok->line_start = tok->buf_offset;
663- tok->start = -1;
664664 }
665665 tok->inp = source_start + chunk.len;
666666 }
667667 else {
668- if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL ) {
668+ if (reset_buffer ) {
669669 tok->buf_offset = tok->inp;
670670 }
671671 tok->inp += chunk.len;
@@ -674,9 +674,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
674674
675675 tok->lineno++;
676676 if (kind == _PYTOK_READER_FILE &&
677- (reader->encoding == NULL || strcmp(reader->encoding, "utf-8") == 0) &&
677+ (reader->encoding == NULL ||
678+ strcmp(reader->encoding, "utf-8") == 0) &&
678679 !_PyTokenizer_ensure_utf8(
679- _PyTok_SourcePointer(&tok->source, tok->cur), tok, tok->lineno)) {
680+ _PyTok_SourcePointer(&tok->source, tok->cur),
681+ tok, tok->lineno)) {
680682 _PyTok_ChunkClear(&chunk);
681683 return 0;
682684 }
0 commit comments