Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions src/jmh/java/benchmark/NumericLexerBenchmark.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package benchmark;

import graphql.parser.InvalidSyntaxException;
import graphql.parser.Parser;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;

import java.util.concurrent.TimeUnit;

@State(Scope.Thread)
@Warmup(iterations = 3, time = 5)
@Measurement(iterations = 3, time = 5)
@Fork(1)
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public class NumericLexerBenchmark {

@Param({"small", "manySmall", "longInteger", "longMalformedExponent"})
public String inputShape;

private String query;

@Setup
public void setup() {
switch (inputShape) {
case "small":
query = "{ f(arg: 12345) }";
return;
case "manySmall":
query = "{ f(arg: [" + "12345,".repeat(1000) + "]) }";
return;
case "longInteger":
query = "{ f(arg: " + "9".repeat(900_000) + ") }";
return;
case "longMalformedExponent":
query = "{ f(arg: 1e" + "9".repeat(900_000) + "x) }";
return;
default:
throw new IllegalArgumentException(inputShape);
}
}

@Benchmark
public void parse(Blackhole blackhole) {
try {
blackhole.consume(Parser.parse(query));
} catch (InvalidSyntaxException expected) {
blackhole.consume(expected);
}
}
}
15 changes: 15 additions & 0 deletions src/main/antlr/Graphql.g4
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,21 @@ import GraphqlSDL, GraphqlOperation, GraphqlCommon;
}

@lexer::members {
private int maxNumericLiteralCharacters = Integer.MAX_VALUE;

public void setMaxNumericLiteralCharacters(int maxNumericLiteralCharacters) {
this.maxNumericLiteralCharacters = maxNumericLiteralCharacters;
}

private boolean numericLiteralLengthWithinLimit() {
int start = _tokenStartCharIndex;
int end = _input.index();
if (end - start > maxNumericLiteralCharacters) {
throw new NumericLiteralTooLongException(_tokenStartLine, _tokenStartCharPositionInLine, maxNumericLiteralCharacters);
}
return true;
}

public boolean isDigit(int c) {
return c >= '0' && c <= '9';
}
Expand Down
5 changes: 3 additions & 2 deletions src/main/antlr/GraphqlCommon.g4
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ NAME: [_A-Za-z][_0-9A-Za-z]*;


// Int Value
IntValue : IntegerPart { !isDigit(_input.LA(1)) && !isDot(_input.LA(1)) && !isNameStart(_input.LA(1)) }?;
// Digit checks longer numeric values while matching; this also covers single-digit integers.
IntValue : IntegerPart { !isDigit(_input.LA(1)) && !isDot(_input.LA(1)) && !isNameStart(_input.LA(1)) && numericLiteralLengthWithinLimit() }?;
fragment IntegerPart : NegativeSign? '0' | NegativeSign? NonZeroDigit Digit*;
fragment NegativeSign : '-';
fragment NonZeroDigit: '1'..'9';
Expand All @@ -107,7 +108,7 @@ fragment FractionalPart: '.' Digit+;
fragment ExponentPart : ExponentIndicator Sign? Digit+;
fragment ExponentIndicator: 'e' | 'E';
fragment Sign: '+'|'-';
fragment Digit : '0'..'9';
fragment Digit : '0'..'9' { numericLiteralLengthWithinLimit() }?;

// StringValue
StringValue:
Expand Down
18 changes: 12 additions & 6 deletions src/main/java/graphql/parser/Parser.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import graphql.parser.antlr.GraphqlBaseListener;
import graphql.parser.antlr.GraphqlLexer;
import graphql.parser.antlr.GraphqlParser;
import graphql.parser.antlr.NumericLiteralTooLongException;
import graphql.parser.exceptions.ParseCancelledException;
import graphql.parser.exceptions.ParseCancelledTooDeepException;
import graphql.parser.exceptions.ParseCancelledTooManyCharsException;
Expand Down Expand Up @@ -240,6 +241,16 @@ private Node<?> parseImpl(ParserEnvironment environment, BiFunction<GraphqlParse

MultiSourceReader multiSourceReader = setupMultiSourceReader(environment, parserOptions);

try {
return parseImpl(environment, nodeFunction, parserOptions, multiSourceReader);
} catch (NumericLiteralTooLongException exception) {
SourceLocation sourceLocation = AntlrHelper.createSourceLocation(multiSourceReader, exception.getLine(), exception.getColumn());
throw new ParseCancelledTooManyNumericLiteralCharactersException(environment.getI18N(), sourceLocation, exception.getMaxCharacters());
}
}

private Node<?> parseImpl(ParserEnvironment environment, BiFunction<GraphqlParser, GraphqlAntlrToLanguage, Object[]> nodeFunction,
ParserOptions parserOptions, MultiSourceReader multiSourceReader) {
SafeTokenReader safeTokenReader = setupSafeTokenReader(environment, parserOptions, multiSourceReader);

CodePointCharStream charStream = setupCharStream(safeTokenReader);
Expand Down Expand Up @@ -394,17 +405,12 @@ private SafeTokenSource getSafeTokenSource(ParserEnvironment environment, Parser
maxTokenCount,
multiSourceReader,
ParseCancelledException.class);
BiConsumer<Integer, Token> onTooManyNumericLiteralCharacters = (maxCharacters, token) -> {
SourceLocation sourceLocation = AntlrHelper.createSourceLocation(multiSourceReader, token);
throw new ParseCancelledTooManyNumericLiteralCharactersException(environment.getI18N(), sourceLocation, maxCharacters);
};
return new SafeTokenSource(
lexer,
maxTokens,
maxWhitespaceTokens,
maxNumericLiteralCharacters,
onTooManyTokens,
onTooManyNumericLiteralCharacters
onTooManyTokens
);
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/graphql/parser/ParserOptions.java
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ public int getMaxWhitespaceTokens() {
/**
* A numeric literal is represented by a single token, regardless of how many characters it contains. Converting
* very large numeric literals into arbitrary precision numbers can consume excessive CPU and memory. This limit
* stops parsing before that conversion takes place.
* stops lexing before a larger literal is fully matched or converted.
*
* @return the maximum number of characters permitted in an integer or floating-point literal
*/
Expand Down Expand Up @@ -416,7 +416,7 @@ public Builder maxWhitespaceTokens(int maxWhitespaceTokens) {

/**
* Sets the maximum number of characters permitted in an integer or floating-point literal. Parsing is
* cancelled before converting a larger literal into an arbitrary precision number.
* cancelled before a larger literal is fully matched or converted into an arbitrary precision number.
*
* @param maxNumericLiteralCharacters the maximum number of characters permitted in a numeric literal
*
Expand Down
25 changes: 4 additions & 21 deletions src/main/java/graphql/parser/SafeTokenSource.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,35 +23,30 @@
@Internal
public class SafeTokenSource implements TokenSource {

private final TokenSource lexer;
private final GraphqlLexer lexer;
private final int maxTokens;
private final int maxWhitespaceTokens;
private final int maxNumericLiteralCharacters;
private final BiConsumer<Integer, Token> whenMaxTokensExceeded;
private final BiConsumer<Integer, Token> whenMaxNumericLiteralCharactersExceeded;
private final int channelCounts[];

public SafeTokenSource(TokenSource lexer, int maxTokens, int maxWhitespaceTokens, int maxNumericLiteralCharacters,
BiConsumer<Integer, Token> whenMaxTokensExceeded,
BiConsumer<Integer, Token> whenMaxNumericLiteralCharactersExceeded) {
public SafeTokenSource(GraphqlLexer lexer, int maxTokens, int maxWhitespaceTokens, int maxNumericLiteralCharacters,
BiConsumer<Integer, Token> whenMaxTokensExceeded) {
this.lexer = lexer;
this.maxTokens = maxTokens;
this.maxWhitespaceTokens = maxWhitespaceTokens;
this.maxNumericLiteralCharacters = maxNumericLiteralCharacters;
this.whenMaxTokensExceeded = whenMaxTokensExceeded;
this.whenMaxNumericLiteralCharactersExceeded = whenMaxNumericLiteralCharactersExceeded;
// this could be a Map<int,int> however we want it to be faster as possible.
// we only have 3 channels - but they are 0,2 and 3 so use 5 for safety - still faster than a map get/put
// if we ever add another channel beyond 5 it will IOBEx during tests so future changes will be handled before release!
this.channelCounts = new int[]{0, 0, 0, 0, 0};
lexer.setMaxNumericLiteralCharacters(maxNumericLiteralCharacters);
}


@Override
public Token nextToken() {
Token token = lexer.nextToken();
if (token != null) {
callbackIfNumericLiteralTooLong(token);
int channel = token.getChannel();
int currentCount = ++channelCounts[channel];
if (channel == Parser.CHANNEL_WHITESPACE) {
Expand All @@ -64,18 +59,6 @@ public Token nextToken() {
return token;
}

private void callbackIfNumericLiteralTooLong(Token token) {
int tokenType = token.getType();
if (tokenType != GraphqlLexer.IntValue && tokenType != GraphqlLexer.FloatValue) {
return;
}

int characterCount = token.getStopIndex() - token.getStartIndex() + 1;
if (characterCount > maxNumericLiteralCharacters) {
whenMaxNumericLiteralCharactersExceeded.accept(maxNumericLiteralCharacters, token);
}
}

private void callbackIfMaxExceeded(int maxCount, int currentCount, Token token) {
if (currentCount > maxCount) {
whenMaxTokensExceeded.accept(maxCount, token);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package graphql.parser.antlr;

import graphql.Internal;

@Internal
public class NumericLiteralTooLongException extends RuntimeException {

private final int line;
private final int column;
private final int maxCharacters;

public NumericLiteralTooLongException(int line, int column, int maxCharacters) {
this.line = line;
this.column = column;
this.maxCharacters = maxCharacters;
}

public int getLine() {
return line;
}

public int getColumn() {
return column;
}

public int getMaxCharacters() {
return maxCharacters;
}
}
71 changes: 71 additions & 0 deletions src/test/groovy/graphql/parser/ParserExceptionTest.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@ package graphql.parser
import graphql.GraphQL
import graphql.InvalidSyntaxError
import graphql.StarWarsSchema
import graphql.TestUtil
import graphql.language.SourceLocation
import graphql.parser.exceptions.ParseCancelledTooManyNumericLiteralCharactersException
import graphql.schema.DataFetcher
import spock.lang.Specification

import static graphql.ExecutionInput.newExecutionInput

class ParserExceptionTest extends Specification {
def badQueryPart1 = '''
query X {
Expand Down Expand Up @@ -113,4 +119,69 @@ fragment X on SomeType {

}

def "numeric limit errors preserve source mapping and locale #locale"() {
given:
def source = MultiSourceReader.newMultiSourceReader()
.string("type Query { f: String }\n", "query.graphqls")
.string("input Filter {\n amount: Int = ${'9' * 101}\n}\n", "filter.graphqls")
.build()
def environment = ParserEnvironment.newParserEnvironment()
.document(source)
.parserOptions(ParserOptions.getDefaultSdlParserOptions())
.locale(locale)
.build()

when:
Parser.parse(environment)

then:
def exception = thrown(ParseCancelledTooManyNumericLiteralCharactersException)
exception.location == new SourceLocation(2, 17, 'filter.graphqls')
exception.message.contains(message)
exception.offendingToken == null

where:
locale | message
Locale.ENGLISH | 'more than 100 characters'
Locale.GERMAN | 'mehr als 100 Zeichen'
}

def "standalone value parsing translates numeric limit errors"() {
when:
Parser.parseValue('9' * 101)

then:
def exception = thrown(ParseCancelledTooManyNumericLiteralCharactersException)
exception.location == new SourceLocation(1, 1)
}

def "execution reports a numeric limit error without fetching data - #description"() {
given:
def fetcher = Mock(DataFetcher)
def graphQL = TestUtil.graphQL('type Query { f(arg: Float): String }', [Query: [f: fetcher]]).build()
def options = ParserOptions.getDefaultOperationParserOptions().transform {
it.maxNumericLiteralCharacters(5)
}
def input = newExecutionInput().query("{\n f(arg: $numericInput)\n}")
.graphQLContext([(ParserOptions): options])
.locale(Locale.ENGLISH)
.build()

when:
def result = graphQL.execute(input)

then:
!result.dataPresent
result.errors.size() == 1
result.errors[0] instanceof InvalidSyntaxError
result.errors[0].message.contains('more than 5 characters')
result.errors[0].locations == [new SourceLocation(2, 10)]
0 * fetcher.get(_)

where:
description | numericInput
'integer' | '123456'
'malformed exponent' | '1e' + '9' * 10_000 + 'x'
}

}
39 changes: 39 additions & 0 deletions src/test/groovy/graphql/parser/ParserStressTest.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,45 @@ class ParserStressTest extends Specification {
"9" * 99 + ".0" | "9" * 100 + ".0"
}

def "a larger numeric character limit is enforced while lexing"() {
given:
def parserOptions = defaultOperationOptions.transform {
it.maxNumericLiteralCharacters(2000)
}
def accepted = newParserEnvironment()
.document("query { f(arg: ${"9" * 2000}) }")
.parserOptions(parserOptions)
.build()

expect:
Parser.parse(accepted) != null

when:
def rejected = newParserEnvironment()
.document("query {\n f(arg: ${"9" * 2001})\n}")
.parserOptions(parserOptions)
.build()
Parser.parse(rejected)

then:
def exception = thrown(ParseCancelledTooManyNumericLiteralCharactersException)
exception.message.contains("more than 2,000 characters")
exception.location.line == 2
exception.location.column == 10
}

def "digits in comments and strings are not subject to the numeric ceiling"() {
given:
def parserOptions = defaultOperationOptions.transform {
it.maxNumericLiteralCharacters(2000)
}
def document = "# ${"9" * 2001}\nquery { f(arg: \"${"9" * 2001}\") }"
def environment = newParserEnvironment().document(document).parserOptions(parserOptions).build()

expect:
Parser.parse(environment) != null
}

String mkDeepQuery(int howMany) {
def field = 'f(a:"")'
StringBuilder sb = new StringBuilder("query q{")
Expand Down
Loading
Loading