Skip to content

Commit 4174d87

Browse files
cushoncushon
committed
Platform-independent input newline handling
Handle CR and CRLF newlines in input. Output is still hard-coded to use LF for now. MOE_MIGRATED_REVID=136407062
1 parent 3caaae5 commit 4174d87

File tree

10 files changed

+369
-79
lines changed

10 files changed

+369
-79
lines changed

‎core/src/main/java/com/google/googlejavaformat/Doc.java‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@
1414

1515
package com.google.googlejavaformat;
1616

17+
import static com.google.common.collect.Iterables.getLast;
18+
1719
import com.google.common.base.MoreObjects;
1820
import com.google.common.base.Optional;
1921
import com.google.common.collect.DiscreteDomain;
22+
import com.google.common.collect.Iterators;
2023
import com.google.common.collect.Range;
2124
import com.google.googlejavaformat.Output.BreakTag;
2225
import java.util.ArrayList;
@@ -274,7 +277,7 @@ private static void splitByBreaks(List<Doc> docs, List<List<Doc>> splits, List<B
274277
breaks.add((Break) doc);
275278
splits.add(new ArrayList<Doc>());
276279
} else {
277-
splits.get(splits.size() - 1).add(doc);
280+
getLast(splits).add(doc);
278281
}
279282
}
280283
}
@@ -714,16 +717,16 @@ public void add(DocBuilder builder) {
714717

715718
@Override
716719
float computeWidth() {
720+
int idx = Newlines.firstBreak(tok.getOriginalText());
717721
// only count the first line of multi-line block comments
718722
if (tok.isComment()) {
719-
int idx = tok.getOriginalText().indexOf('\n');
720723
if (idx > 0) {
721724
return idx;
722725
} else {
723726
return tok.length();
724727
}
725728
}
726-
return tok.getOriginalText().contains("\n") ? Float.POSITIVE_INFINITY : (float) tok.length();
729+
return idx != -1 ? Float.POSITIVE_INFINITY : (float) tok.length();
727730
}
728731

729732
@Override
@@ -740,19 +743,8 @@ Range<Integer> computeRange() {
740743

741744
@Override
742745
public State computeBreaks(CommentsHelper commentsHelper, int maxWidth, State state) {
743-
int column = state.column;
744-
int lines = 0;
745-
text = commentsHelper.rewrite(tok, maxWidth, column);
746-
// TODO(lowasser): use lastIndexOf('\n')
747-
for (char c : text.toCharArray()) {
748-
if (c == '\n') {
749-
column = 0;
750-
lines++;
751-
} else {
752-
column++;
753-
}
754-
}
755-
return state.withColumn(column);
746+
text = commentsHelper.rewrite(tok, maxWidth, state.column);
747+
return state.withColumn(text.length() - Iterators.getLast(Newlines.lineOffsetIterator(text)));
756748
}
757749

758750
@Override

‎core/src/main/java/com/google/googlejavaformat/InputOutput.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
package com.google.googlejavaformat;
1616

17-
import com.google.common.base.CharMatcher;
1817
import com.google.common.collect.DiscreteDomain;
1918
import com.google.common.collect.ImmutableList;
2019
import com.google.common.collect.Range;
@@ -28,7 +27,6 @@ public abstract class InputOutput {
2827
private ImmutableList<String> lines = ImmutableList.of();
2928

3029
protected static final Range<Integer> EMPTY_RANGE = Range.closedOpen(-1, -1);
31-
private static final CharMatcher NEWLINE_MATCHER = CharMatcher.is('\n');
3230
private static final DiscreteDomain<Integer> INTEGERS = DiscreteDomain.integers();
3331

3432
/** Set the lines. */
@@ -77,7 +75,7 @@ protected final void computeRanges(List<? extends Input.Tok> toks) {
7775
for (Input.Tok tok : toks) {
7876
String txt = tok.getOriginalText();
7977
int lineI0 = lineI;
80-
lineI += NEWLINE_MATCHER.countIn(txt);
78+
lineI += Newlines.count(txt);
8179
int k = tok.getIndex();
8280
if (k >= 0) {
8381
addToRanges(range0s, lineI0, k);

‎core/src/main/java/com/google/googlejavaformat/Newlines.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Copyright 2016 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5+
* in compliance with the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License
10+
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11+
* or implied. See the License for the specific language governing permissions and limitations under
12+
* the License.
13+
*/
14+
15+
package com.google.googlejavaformat;
16+
17+
import com.google.common.base.CharMatcher;
18+
import com.google.common.collect.ImmutableSet;
19+
import com.google.common.collect.Iterators;
20+
import java.util.Iterator;
21+
import java.util.NoSuchElementException;
22+
23+
/** Platform-independent newline handling. */
24+
public class Newlines {
25+
26+
/** Returns the number of line breaks in the input. */
27+
public static int count(String input) {
28+
return Iterators.size(lineOffsetIterator(input)) - 1;
29+
}
30+
31+
/** Returns the index of the first break in the input, or {@code -1}. */
32+
public static int firstBreak(String input) {
33+
Iterator<Integer> it = lineOffsetIterator(input);
34+
it.next();
35+
return it.hasNext() ? it.next() : -1;
36+
}
37+
38+
private static final ImmutableSet<String> BREAKS = ImmutableSet.of("\r\n", "\n", "\r");
39+
40+
/** Returns true if the entire input string is a recognized line break. */
41+
public static boolean isNewline(String input) {
42+
return BREAKS.contains(input);
43+
}
44+
45+
/**
46+
* Returns the terminating line break in the input, or {@code null} if the input does not end in a
47+
* break.
48+
*/
49+
public static String getLineEnding(String input) {
50+
for (String b : BREAKS) {
51+
if (input.endsWith(b)) {
52+
return b;
53+
}
54+
}
55+
return null;
56+
}
57+
58+
/** Returns true if the input contains any line breaks. */
59+
public static boolean containsBreaks(String text) {
60+
return CharMatcher.anyOf("\n\r").matchesAnyOf(text);
61+
}
62+
63+
/** Returns an iterator over the start offsets of lines in the input. */
64+
public static Iterator<Integer> lineOffsetIterator(String input) {
65+
return new LineOffsetIterator(input);
66+
}
67+
68+
/** Returns an iterator over lines in the input, including trailing whitespace. */
69+
public static Iterator<String> lineIterator(String input) {
70+
return new LineIterator(input);
71+
}
72+
73+
private static class LineOffsetIterator implements Iterator<Integer> {
74+
75+
private int curr = 0;
76+
private int idx = 0;
77+
private final String input;
78+
79+
private LineOffsetIterator(String input) {
80+
this.input = input;
81+
}
82+
83+
@Override
84+
public boolean hasNext() {
85+
return curr != -1;
86+
}
87+
88+
@Override
89+
public Integer next() {
90+
if (curr == -1) {
91+
throw new NoSuchElementException();
92+
}
93+
int result = curr;
94+
advance();
95+
return result;
96+
}
97+
98+
private void advance() {
99+
for (; idx < input.length(); idx++) {
100+
char c = input.charAt(idx);
101+
switch (c) {
102+
case '\r':
103+
if (idx + 1 < input.length() && input.charAt(idx + 1) == '\n') {
104+
idx++;
105+
}
106+
// falls through
107+
case '\n':
108+
idx++;
109+
curr = idx;
110+
return;
111+
default:
112+
break;
113+
}
114+
}
115+
curr = -1;
116+
}
117+
118+
@Override
119+
public void remove() {
120+
throw new UnsupportedOperationException("remove");
121+
}
122+
}
123+
124+
private static class LineIterator implements Iterator<String> {
125+
126+
int idx;
127+
String curr;
128+
129+
private final String input;
130+
private final Iterator<Integer> indices;
131+
132+
private LineIterator(String input) {
133+
this.input = input;
134+
this.indices = lineOffsetIterator(input);
135+
idx = indices.next(); // read leading 0
136+
}
137+
138+
private void advance() {
139+
int last = idx;
140+
if (indices.hasNext()) {
141+
idx = indices.next();
142+
} else if (hasNext()) {
143+
// no terminal line break
144+
idx = input.length();
145+
} else {
146+
throw new NoSuchElementException();
147+
}
148+
curr = input.substring(last, idx);
149+
}
150+
151+
@Override
152+
public boolean hasNext() {
153+
return idx < input.length();
154+
}
155+
156+
@Override
157+
public String next() {
158+
advance();
159+
return curr;
160+
}
161+
162+
@Override
163+
public void remove() {
164+
throw new UnsupportedOperationException("remove");
165+
}
166+
}
167+
}

‎core/src/main/java/com/google/googlejavaformat/java/Formatter.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@
1616

1717
import static java.nio.charset.StandardCharsets.UTF_8;
1818

19-
import com.google.common.base.CharMatcher;
2019
import com.google.common.base.Predicate;
2120
import com.google.common.collect.ImmutableList;
2221
import com.google.common.collect.Iterables;
22+
import com.google.common.collect.Iterators;
2323
import com.google.common.collect.Range;
2424
import com.google.common.collect.RangeSet;
2525
import com.google.common.collect.TreeRangeSet;
@@ -29,6 +29,7 @@
2929
import com.google.googlejavaformat.Doc;
3030
import com.google.googlejavaformat.DocBuilder;
3131
import com.google.googlejavaformat.FormattingError;
32+
import com.google.googlejavaformat.Newlines;
3233
import com.google.googlejavaformat.Op;
3334
import com.google.googlejavaformat.OpsBuilder;
3435
import com.sun.tools.javac.file.JavacFileManager;
@@ -242,20 +243,13 @@ public ImmutableList<Replacement> getFormatReplacements(
242243
return javaOutput.getFormatReplacements(tokenRangeSet);
243244
}
244245

245-
static final CharMatcher NEWLINE = CharMatcher.is('\n');
246-
247246
/**
248247
* Converts zero-indexed, [closed, open) line ranges in the given source file to character ranges.
249248
*/
250249
public static RangeSet<Integer> lineRangesToCharRanges(
251250
String input, RangeSet<Integer> lineRanges) {
252251
List<Integer> lines = new ArrayList<>();
253-
lines.add(0);
254-
int idx = NEWLINE.indexIn(input);
255-
while (idx >= 0) {
256-
lines.add(idx + 1);
257-
idx = NEWLINE.indexIn(input, idx + 1);
258-
}
252+
Iterators.addAll(lines, Newlines.lineOffsetIterator(input));
259253
lines.add(input.length() + 1);
260254

261255
final RangeSet<Integer> characterRanges = TreeRangeSet.create();

‎core/src/main/java/com/google/googlejavaformat/java/ImportOrderer.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
package com.google.googlejavaformat.java;
1515

16+
import static com.google.common.collect.Iterables.getLast;
1617
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameclass;
1718
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameenum;
1819
import static org.eclipse.jdt.core.compiler.ITerminalSymbols.TokenNameinterface;
@@ -124,7 +125,7 @@ private String reorderImports() throws FormatterException {
124125
if (toks.isEmpty()) {
125126
tail = "";
126127
} else {
127-
Tok lastTok = toks.get(toks.size() - 1);
128+
Tok lastTok = getLast(toks);
128129
int tailStart = lastTok.getPosition() + lastTok.length();
129130
tail = text.substring(tailStart);
130131
}
@@ -336,8 +337,7 @@ private boolean isSpaceToken(int i) {
336337
if (s.isEmpty()) {
337338
return false;
338339
} else {
339-
// TODO(b/26984991): if the formatter starts understanding \r\n then \r should be removed here
340-
return " \t\f\r".indexOf(s.codePointAt(0)) >= 0;
340+
return " \t\f".indexOf(s.codePointAt(0)) >= 0;
341341
}
342342
}
343343

‎core/src/main/java/com/google/googlejavaformat/java/JavaCommentsHelper.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515
package com.google.googlejavaformat.java;
1616

1717
import com.google.common.base.CharMatcher;
18-
import com.google.common.base.Splitter;
1918
import com.google.common.base.Strings;
2019
import com.google.googlejavaformat.CommentsHelper;
2120
import com.google.googlejavaformat.Input.Tok;
21+
import com.google.googlejavaformat.Newlines;
2222
import com.google.googlejavaformat.java.javadoc.JavadocFormatter;
2323
import java.util.ArrayList;
2424
import java.util.Iterator;
@@ -27,8 +27,6 @@
2727
/** {@code JavaCommentsHelper} extends {@link CommentsHelper} to rewrite Java comments. */
2828
public final class JavaCommentsHelper implements CommentsHelper {
2929

30-
private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n');
31-
3230
private final JavaFormatterOptions options;
3331

3432
public JavaCommentsHelper(JavaFormatterOptions options) {
@@ -45,8 +43,9 @@ public String rewrite(Tok tok, int maxWidth, int column0) {
4543
text = JavadocFormatter.formatJavadoc(text, column0, options);
4644
}
4745
List<String> lines = new ArrayList<>();
48-
for (String line : NEWLINE_SPLITTER.split(text)) {
49-
lines.add(CharMatcher.whitespace().trimTrailingFrom(line));
46+
Iterator<String> it = Newlines.lineIterator(text);
47+
while (it.hasNext()) {
48+
lines.add(CharMatcher.whitespace().trimTrailingFrom(it.next()));
5049
}
5150
if (tok.isSlashSlashComment()) {
5251
return indentLineComments(lines, column0);

0 commit comments

Comments
 (0)