|
17 | 17 | import graphql.parser.exceptions.ParseCancelledTooManyCharsException; |
18 | 18 | import graphql.parser.exceptions.ParseCancelledTooManyNumericLiteralCharactersException; |
19 | 19 | import org.antlr.v4.runtime.BaseErrorListener; |
20 | | -import org.antlr.v4.runtime.CharStreams; |
| 20 | +import org.antlr.v4.runtime.CodePointBuffer; |
21 | 21 | import org.antlr.v4.runtime.CodePointCharStream; |
22 | 22 | import org.antlr.v4.runtime.CommonTokenStream; |
| 23 | +import org.antlr.v4.runtime.IntStream; |
23 | 24 | import org.antlr.v4.runtime.ParserRuleContext; |
24 | 25 | import org.antlr.v4.runtime.RecognitionException; |
25 | 26 | import org.antlr.v4.runtime.Recognizer; |
|
32 | 33 | import java.io.IOException; |
33 | 34 | import java.io.Reader; |
34 | 35 | import java.io.UncheckedIOException; |
| 36 | +import java.nio.CharBuffer; |
35 | 37 | import java.util.List; |
36 | 38 | import java.util.Optional; |
37 | 39 | import java.util.function.BiConsumer; |
@@ -309,15 +311,51 @@ private static SafeTokenReader setupSafeTokenReader(ParserEnvironment environmen |
309 | 311 | return new SafeTokenReader(multiSourceReader, maxCharacters, onTooManyCharacters); |
310 | 312 | } |
311 | 313 |
|
| 314 | + // matches the chunk size ANTLR's CharStreams.fromReader() uses internally |
| 315 | + private static final int CHAR_STREAM_BUFFER_SIZE = 4096; |
| 316 | + |
312 | 317 | @NonNull |
313 | 318 | private static CodePointCharStream setupCharStream(SafeTokenReader safeTokenReader) { |
314 | | - CodePointCharStream charStream; |
315 | 319 | try { |
316 | | - charStream = CharStreams.fromReader(safeTokenReader); |
| 320 | + return readCodePointCharStream(safeTokenReader, CHAR_STREAM_BUFFER_SIZE); |
317 | 321 | } catch (IOException e) { |
318 | 322 | throw new UncheckedIOException(e); |
319 | 323 | } |
320 | | - return charStream; |
| 324 | + } |
| 325 | + |
| 326 | + /** |
| 327 | + * This mirrors {@link CharStreams#fromReader(Reader)}, except that a surrogate pair split across |
| 328 | + * two chunk reads is held back and carried into the next chunk instead of being mis-decoded. |
| 329 | + * <p> |
| 330 | + * ANTLR's {@link CodePointBuffer.Builder#append(CharBuffer)} keeps a pending high surrogate across |
| 331 | + * calls, but its flush at the end of a chunk fails to clear that pending state, so a surrogate pair |
| 332 | + * split across two {@code append()} calls gets decoded into two separate code points instead of one. |
| 333 | + * Never handing it a chunk that ends on an unpaired high surrogate avoids the bug entirely. |
| 334 | + * |
| 335 | + * @see <a href="https://github.com/antlr/antlr4/pull/4943">antlr/antlr4#4943</a> |
| 336 | + */ |
| 337 | + private static CodePointCharStream readCodePointCharStream(Reader reader, int bufferSize) throws IOException { |
| 338 | + CodePointBuffer.Builder builder = CodePointBuffer.builder(bufferSize); |
| 339 | + CharBuffer charBuffer = CharBuffer.allocate(bufferSize); |
| 340 | + try { |
| 341 | + while (reader.read(charBuffer) != -1) { |
| 342 | + charBuffer.flip(); |
| 343 | + int realLimit = charBuffer.limit(); |
| 344 | + if (realLimit > charBuffer.position() && Character.isHighSurrogate(charBuffer.get(realLimit - 1))) { |
| 345 | + charBuffer.limit(realLimit - 1); |
| 346 | + } |
| 347 | + builder.append(charBuffer); |
| 348 | + charBuffer.limit(realLimit); |
| 349 | + charBuffer.compact(); |
| 350 | + } |
| 351 | + charBuffer.flip(); |
| 352 | + if (charBuffer.hasRemaining()) { |
| 353 | + builder.append(charBuffer); |
| 354 | + } |
| 355 | + return CodePointCharStream.fromBuffer(builder.build(), IntStream.UNKNOWN_SOURCE_NAME); |
| 356 | + } finally { |
| 357 | + reader.close(); |
| 358 | + } |
321 | 359 | } |
322 | 360 |
|
323 | 361 | @NonNull |
|
0 commit comments