Skip to content

Commit 0d6671d

Browse files
authored
gh-153568: Skip newline translation for source without carriage returns (#153584)
The tokenizer copied every input byte by byte to normalize newlines, but source without a carriage return needs no translation and can be copied verbatim.
1 parent f88d668 commit 0d6671d

2 files changed

Lines changed: 16 additions & 7 deletions

File tree

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Speed up the tokenizer by copying source text verbatim when it contains no
2+
carriage returns.

Parser/tokenizer/decoder.c

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,22 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf,
8787
return NULL;
8888
}
8989
Py_ssize_t write = 0;
90-
for (Py_ssize_t read = 0; read < len; read++) {
91-
char c = data[read];
92-
if (!preserve_crlf && c == '\r') {
93-
if (read + 1 < len && data[read + 1] == '\n') {
94-
read++;
90+
if (memchr(data, '\r', len) == NULL) {
91+
// No carriage returns: nothing to translate, copy verbatim.
92+
memcpy(result, data, len);
93+
write = len;
94+
}
95+
else {
96+
for (Py_ssize_t read = 0; read < len; read++) {
97+
char c = data[read];
98+
if (!preserve_crlf && c == '\r') {
99+
if (read + 1 < len && data[read + 1] == '\n') {
100+
read++;
101+
}
102+
c = '\n';
95103
}
96-
c = '\n';
104+
result[write++] = c;
97105
}
98-
result[write++] = c;
99106
}
100107
int implicit = add_final_newline && write > 0 && result[write - 1] != '\n';
101108
if (implicit) {

0 commit comments

Comments
 (0)