Skip to content

Commit 434682a

Browse files
committed
Fix CR line tracking and block string normalization
1 parent 116cd47 commit 434682a

6 files changed

Lines changed: 268 additions & 8 deletions

File tree

‎src/main/antlr/GraphqlCommon.g4‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,6 @@ Comment: '#' SourceCharacterWithoutLineFeed* -> channel(2);
143143
144144
LF: [\n] -> channel(3);
145145
CR: [\r] -> channel(3);
146-
LineTerminator: [\u2028\u2029] -> channel(3);
147146
148147
Space : [\u0020] -> channel(3);
149148
Tab : [\u0009] -> channel(3);
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package graphql.parser;
2+
3+
import graphql.Internal;
4+
import org.antlr.v4.runtime.CharStream;
5+
import org.antlr.v4.runtime.Lexer;
6+
import org.antlr.v4.runtime.atn.LexerATNSimulator;
7+
import org.jspecify.annotations.NullMarked;
8+
9+
@Internal
10+
@NullMarked
11+
public class GraphqlLexerATNSimulator extends LexerATNSimulator {
12+
13+
public GraphqlLexerATNSimulator(Lexer lexer) {
14+
super(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, lexer.getInterpreter().getSharedContextCache());
15+
}
16+
17+
@Override
18+
public void consume(CharStream input) {
19+
int codePoint = input.LA(1);
20+
// ANTLR only counts LF. GraphQL also counts a standalone CR, while CRLF counts once.
21+
if (codePoint == '\n' || (codePoint == '\r' && input.LA(2) != '\n')) {
22+
setLine(getLine() + 1);
23+
setCharPositionInLine(0);
24+
} else {
25+
setCharPositionInLine(getCharPositionInLine() + 1);
26+
}
27+
input.consume();
28+
}
29+
}

‎src/main/java/graphql/parser/Parser.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ private static CodePointCharStream readCodePointCharStream(Reader reader, int bu
361361
@NonNull
362362
private static GraphqlLexer setupGraphqlLexer(ParserEnvironment environment, MultiSourceReader multiSourceReader, CodePointCharStream charStream) {
363363
GraphqlLexer lexer = new GraphqlLexer(charStream);
364+
lexer.setInterpreter(new GraphqlLexerATNSimulator(lexer));
364365
lexer.removeErrorListeners();
365366
lexer.addErrorListener(new BaseErrorListener() {
366367
@Override

‎src/main/java/graphql/parser/StringValueParsing.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@
55
import graphql.i18n.I18n;
66
import graphql.language.SourceLocation;
77

8+
import java.util.regex.Pattern;
9+
810
/**
911
* Contains parsing code for the StringValue types in the grammar
1012
*/
1113
@Internal
1214
public class StringValueParsing {
1315
private final static String ESCAPED_TRIPLE_QUOTES = "\\\"\"\"";
1416
private final static String THREE_QUOTES = "\"\"\"";
17+
private static final Pattern LINE_TERMINATORS = Pattern.compile("\\r\\n|[\\r\\n]");
1518

1619
public static String parseTripleQuotedString(String strText) {
1720
int end = strText.length() - 3;
@@ -24,7 +27,7 @@ public static String parseTripleQuotedString(String strText) {
2427
See https://github.com/facebook/graphql/pull/327/files#diff-fe406b08746616e2f5f00909488cce66R758
2528
*/
2629
public static String removeIndentation(String rawValue) {
27-
String[] lines = rawValue.split("\n");
30+
String[] lines = LINE_TERMINATORS.split(rawValue);
2831
Integer commonIndent = null;
2932
for (int i = 0; i < lines.length; i++) {
3033
if (i == 0) {
Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
package graphql.parser
2+
3+
import graphql.TestUtil
4+
import graphql.language.Field
5+
import graphql.language.OperationDefinition
6+
import graphql.language.SourceLocation
7+
import graphql.language.StringValue
8+
import spock.lang.Specification
9+
10+
class LineTerminatorTest extends Specification {
11+
12+
def "AST locations and comments support #label"() {
13+
given:
14+
def query = ['{', ' first # comment 😀', ' second', '}'].join(lineTerminator)
15+
16+
when:
17+
def document = new Parser().parseDocument(new StringReader(query))
18+
def operation = document.getDefinitionsOfType(OperationDefinition)[0]
19+
def fields = operation.selectionSet.getSelectionsOfType(Field)
20+
21+
then:
22+
fields[0].sourceLocation == new SourceLocation(2, 3)
23+
fields[1].sourceLocation == new SourceLocation(3, 3)
24+
fields[1].comments*.content == [' comment 😀']
25+
26+
where:
27+
label | lineTerminator
28+
'LF' | '\n'
29+
'CR' | '\r'
30+
'CRLF' | '\r\n'
31+
}
32+
33+
def "syntax errors after #label retain code point columns"() {
34+
when:
35+
Parser.parse('{ f(arg: "😀")' + lineTerminator + ' f(arg: "😀") ) }')
36+
37+
then:
38+
def error = thrown(InvalidSyntaxException)
39+
error.location == new SourceLocation(2, 15)
40+
41+
where:
42+
label | lineTerminator
43+
'LF' | '\n'
44+
'CR' | '\r'
45+
'CRLF' | '\r\n'
46+
}
47+
48+
def "lexer errors after #label have the right location"() {
49+
when:
50+
Parser.parse('{ first' + lineTerminator + ' ? }')
51+
52+
then:
53+
def error = thrown(InvalidSyntaxException)
54+
error.location == new SourceLocation(2, 3)
55+
56+
where:
57+
label | lineTerminator
58+
'LF' | '\n'
59+
'CR' | '\r'
60+
'CRLF' | '\r\n'
61+
}
62+
63+
def "EOF after a comment and #label starts a new line"() {
64+
when:
65+
Parser.parse('{ first # comment' + lineTerminator)
66+
67+
then:
68+
def error = thrown(InvalidSyntaxException)
69+
error.location == new SourceLocation(2, 1)
70+
71+
where:
72+
label | lineTerminator
73+
'LF' | '\n'
74+
'CR' | '\r'
75+
'CRLF' | '\r\n'
76+
}
77+
78+
def "block string locations support #label"() {
79+
when:
80+
Parser.parse('{ f(arg: """first' + lineTerminator + 'second""")' + lineTerminator + ' ) }')
81+
82+
then:
83+
def error = thrown(InvalidSyntaxException)
84+
error.location == new SourceLocation(3, 3)
85+
86+
where:
87+
label | lineTerminator
88+
'LF' | '\n'
89+
'CR' | '\r'
90+
'CRLF' | '\r\n'
91+
}
92+
93+
def "block string values normalize #label before removing indentation"() {
94+
given:
95+
def input = ['"""', ' first', ' second', '', ' third', '"""'].join(lineTerminator)
96+
97+
when:
98+
def value = Parser.parseValue(input) as StringValue
99+
100+
then:
101+
value.value == 'first\n second\n\nthird'
102+
103+
where:
104+
label | lineTerminator
105+
'LF' | '\n'
106+
'CR' | '\r'
107+
'CRLF' | '\r\n'
108+
}
109+
110+
def "captured ignored characters preserve #label"() {
111+
given:
112+
def options = ParserOptions.newParserOptions().captureIgnoredChars(true).build()
113+
def environment = ParserEnvironment.newParserEnvironment()
114+
.document('{' + lineTerminator + ' first }')
115+
.parserOptions(options)
116+
.build()
117+
118+
when:
119+
def document = Parser.parse(environment)
120+
def field = document.getDefinitionsOfType(OperationDefinition)[0].selectionSet.getSelectionsOfType(Field)[0]
121+
122+
then:
123+
field.sourceLocation == new SourceLocation(2, 3)
124+
field.ignoredChars.left*.value.join() == lineTerminator + ' '
125+
126+
where:
127+
label | lineTerminator
128+
'LF' | '\n'
129+
'CR' | '\r'
130+
'CRLF' | '\r\n'
131+
}
132+
133+
def "multi source error locations support #label"() {
134+
given:
135+
def reader = MultiSourceReader.newMultiSourceReader()
136+
.string('{ first }' + lineTerminator, 'first')
137+
.string('{' + lineTerminator + ' )' + lineTerminator + '}', 'second')
138+
.build()
139+
140+
when:
141+
new Parser().parseDocument(reader)
142+
143+
then:
144+
def error = thrown(InvalidSyntaxException)
145+
error.location == new SourceLocation(2, 3, 'second')
146+
147+
where:
148+
label | lineTerminator
149+
'LF' | '\n'
150+
'CR' | '\r'
151+
'CRLF' | '\r\n'
152+
}
153+
154+
def "mixed and consecutive line terminators count CRLF once"() {
155+
when:
156+
Parser.parse('# comment\r\r\n\n\r{ ) }')
157+
158+
then:
159+
def error = thrown(InvalidSyntaxException)
160+
error.location == new SourceLocation(5, 3)
161+
}
162+
163+
def "CRLF split across reader buffers counts once"() {
164+
when:
165+
Parser.parse('#' + ('a' * 4094) + '\r\n{ ) }')
166+
167+
then:
168+
def error = thrown(InvalidSyntaxException)
169+
error.location == new SourceLocation(2, 3)
170+
}
171+
172+
def "validation errors use CR line boundaries"() {
173+
given:
174+
def graphQL = TestUtil.graphQL('type Query { first: String }').build()
175+
176+
when:
177+
def result = graphQL.execute('{ first\r unknown }')
178+
179+
then:
180+
result.errors.size() == 1
181+
result.errors[0].locations == [new SourceLocation(2, 3)]
182+
}
183+
184+
def "Unicode separator #codePoint is not ignored between tokens"() {
185+
when:
186+
Parser.parse('{ first' + separator + ' second }')
187+
188+
then:
189+
def error = thrown(InvalidSyntaxException)
190+
error.location == new SourceLocation(1, 8)
191+
192+
where:
193+
codePoint | separator
194+
'U+2028' | '\u2028'
195+
'U+2029' | '\u2029'
196+
}
197+
198+
def "Unicode separator #codePoint remains valid in strings and comments"() {
199+
given:
200+
def text = 'before' + separator + 'after'
201+
202+
expect:
203+
(Parser.parseValue('"' + text + '"') as StringValue).value == text
204+
(Parser.parseValue('"""' + text + '"""') as StringValue).value == text
205+
206+
when:
207+
def document = Parser.parse('#' + text + '\n{ first }')
208+
def operation = document.getDefinitionsOfType(OperationDefinition)[0]
209+
210+
then:
211+
operation.sourceLocation == new SourceLocation(2, 1)
212+
operation.comments*.content == [text]
213+
214+
where:
215+
codePoint | separator
216+
'U+2028' | '\u2028'
217+
'U+2029' | '\u2029'
218+
}
219+
220+
def "escaped line terminators in ordinary strings retain their value"() {
221+
expect:
222+
(Parser.parseValue('"\\r\\n"') as StringValue).value == '\r\n'
223+
}
224+
}

‎src/test/groovy/graphql/parser/ParserTest.groovy‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -806,16 +806,16 @@ triple3 : """edge cases \\""" "" " \\"" \\" edge cases"""
806806
field.getIgnoredChars().getLeft().size() == 3
807807
field.getIgnoredChars().getLeft()[0] == new IgnoredChar(",", IgnoredChar.IgnoredCharKind.COMMA, new SourceLocation(1, 2))
808808
field.getIgnoredChars().getLeft()[1] == new IgnoredChar("\r", IgnoredChar.IgnoredCharKind.CR, new SourceLocation(1, 3))
809-
field.getIgnoredChars().getLeft()[2] == new IgnoredChar(" ", IgnoredChar.IgnoredCharKind.SPACE, new SourceLocation(1, 4))
809+
field.getIgnoredChars().getLeft()[2] == new IgnoredChar(" ", IgnoredChar.IgnoredCharKind.SPACE, new SourceLocation(2, 1))
810810

811811
field.getIgnoredChars().getRight().size() == 2
812-
field.getIgnoredChars().getRight()[0] == new IgnoredChar("\n", IgnoredChar.IgnoredCharKind.LF, new SourceLocation(1, 7))
813-
field.getIgnoredChars().getRight()[1] == new IgnoredChar("\t", IgnoredChar.IgnoredCharKind.TAB, new SourceLocation(2, 1))
812+
field.getIgnoredChars().getRight()[0] == new IgnoredChar("\n", IgnoredChar.IgnoredCharKind.LF, new SourceLocation(2, 4))
813+
field.getIgnoredChars().getRight()[1] == new IgnoredChar("\t", IgnoredChar.IgnoredCharKind.TAB, new SourceLocation(3, 1))
814814

815815
document.getIgnoredChars().getRight().size() == 3
816-
document.getIgnoredChars().getRight()[0] == new IgnoredChar(" ", IgnoredChar.IgnoredCharKind.SPACE, new SourceLocation(2, 3))
817-
document.getIgnoredChars().getRight()[1] == new IgnoredChar(",", IgnoredChar.IgnoredCharKind.COMMA, new SourceLocation(2, 4))
818-
document.getIgnoredChars().getRight()[2] == new IgnoredChar("\n", IgnoredChar.IgnoredCharKind.LF, new SourceLocation(2, 5))
816+
document.getIgnoredChars().getRight()[0] == new IgnoredChar(" ", IgnoredChar.IgnoredCharKind.SPACE, new SourceLocation(3, 3))
817+
document.getIgnoredChars().getRight()[1] == new IgnoredChar(",", IgnoredChar.IgnoredCharKind.COMMA, new SourceLocation(3, 4))
818+
document.getIgnoredChars().getRight()[2] == new IgnoredChar("\n", IgnoredChar.IgnoredCharKind.LF, new SourceLocation(3, 5))
819819
}
820820

821821
def "parsed float with positive exponent"() {
@@ -1092,6 +1092,7 @@ triple3 : """edge cases \\""" "" " \\"" \\" edge cases"""
10921092
}
10931093

10941094
def "ignored chars can be set on or off"() {
1095+
def originalOptions = ParserOptions.getDefaultParserOptions()
10951096
def s = '''
10961097
10971098
type X {
@@ -1138,6 +1139,9 @@ triple3 : """edge cases \\""" "" " \\"" \\" edge cases"""
11381139
type.getIgnoredChars() != IgnoredChars.EMPTY
11391140
!type.getIgnoredChars().getLeft().isEmpty()
11401141
!type.getIgnoredChars().getRight().isEmpty()
1142+
1143+
cleanup:
1144+
ParserOptions.setDefaultParserOptions(originalOptions)
11411145
}
11421146

11431147
def "allow braced escaped unicode"() {

0 commit comments

Comments
 (0)