forked from processing/processing
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathTokenMarkerState.java
More file actions
266 lines (236 loc) · 8.45 KB
/
Copy pathTokenMarkerState.java
File metadata and controls
266 lines (236 loc) · 8.45 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
package processing.app.syntax;
import javax.swing.text.Segment;
/**
* This class serves only as a workaround to preserve API and should be removed
* in the next major version. Base TokenMarker which serves as superclass for
* token markes for various modes is stateful, but single instance was shared
* between all tabs and Editors. This caused inherent bugs by leaking state
* between contexts.
*
* TokenMarker subclasses now serve two purposes: they keep keyword list and
* they override markTokensImpl to provide the marking logic.
*
* Since each tab and Editor should have it's own token marker state, I extracted
* most of the fields and associated metods working with them from TokenMarker
* into this class, and allowed TokenMarker to create instances of this class
* when requested.
*
* The way marking logic is handled is unfortunate, since markTokensImpl is
* expected to call addToken() of TokenMarker superclass instead of – for
* example - returning a List of tokens. I worked around this by plugging in
* TokenMarkerState instance as listener before markTokensImpl is called.
* This behavior is safe since TokenMarker is always operated from Event
* Dispatch Thread and no multithreading is involved.
*
* This allows having only single instance of TokenMarker in a way it was
* intended before while keeping state separate for each tab.
*
* In the next major version TokenMarker shound be redesigned with following
* requirements in mind:
* - Single instance of keyword list and other common data, initialized by Mode
* - Each tab should have its own instance of TokenMarker containing its state
* - Support multiple flavors for different doc types
* - Other modes should provide logic in a way which is compatible with
* multiple states (pure function? Function object?).
* Currently state, logic and keywords list are tied together into one
* TokenMarker instance, which leads to need for this workaround.
*/
public class TokenMarkerState {
protected TokenMarker marker;
protected TokenMarkerState(TokenMarker marker) {
this.marker = marker;
}
/**
* The first token in the list. This should be used as the return
* value from <code>markTokens()</code>.
*/
protected Token firstToken;
/**
* The last token in the list. New tokens are added here.
* This should be set to null before a new line is to be tokenized.
*/
protected Token lastToken;
/**
* An array for storing information about lines. It is enlarged and
* shrunk automatically by the <code>insertLines()</code> and
* <code>deleteLines()</code> methods.
*/
protected byte[] lineInfo;
/**
* The number of lines in the model being tokenized. This can be
* less than the length of the <code>lineInfo</code> array.
*/
protected int length;
/**
* The last tokenized line.
*/
protected int lastLine = -1;
/**
* True if the next line should be painted.
*/
protected boolean nextLineRequested;
/**
* A wrapper for the lower-level <code>markTokensImpl</code> method
* that is called to split a line up into tokens.
*
* @param line The line
* @param lineIndex The line number
*/
public Token markTokens(Segment line, int lineIndex) {
if (lineIndex >= length) {
throw new IllegalArgumentException("Tokenizing invalid line: "
+ lineIndex);
}
marker.setTokenListener(this::addToken);
lastToken = null;
byte prev = (lineIndex == 0) ? Token.NULL : lineInfo[lineIndex - 1];
byte oldToken = lineInfo[lineIndex];
byte token = marker.markTokensImpl(prev, line, lineIndex);
marker.setTokenListener(null);
lineInfo[lineIndex] = token;
/*
* This is a foul hack. It stops nextLineRequested
* from being cleared if the same line is marked twice.
*
* Why is this necessary? It's all JEditTextArea's fault.
* When something is inserted into the text, firing a
* document event, the insertUpdate() method shifts the
* caret (if necessary) by the amount inserted.
*
* All caret movement is handled by the select() method,
* which eventually pipes the new position to scrollTo()
* and calls repaint().
*
* Note that at this point in time, the new line hasn't
* yet been painted; the caret is moved first.
*
* scrollTo() calls offsetToX(), which tokenizes the line
* unless it is being called on the last line painted
* (in which case it uses the text area's painter cached
* token list). What scrollTo() does next is irrelevant.
*
* After scrollTo() has done it's job, repaint() is
* called, and eventually we end up in paintLine(), whose
* job is to paint the changed line. It, too, calls
* markTokens().
*
* The problem was that if the line started a multiline
* token, the first markTokens() (done in offsetToX())
* would set nextLineRequested (because the line end
* token had changed) but the second would clear it
* (because the line was the same that time) and therefore
* paintLine() would never know that it needed to repaint
* subsequent lines.
*
* This bug took me ages to track down, that's why I wrote
* all the relevant info down so that others wouldn't
* duplicate it.
*/
if (!(lastLine == lineIndex && nextLineRequested)) {
nextLineRequested = (oldToken != token);
}
lastLine = lineIndex;
addToken(0, Token.END);
return firstToken;
}
/**
* Informs the token marker that lines have been inserted into
* the document. This inserts a gap in the <code>lineInfo</code>
* array.
*
* @param index The first line number
* @param lines The number of lines
*/
public void insertLines(int index, int lines) {
if (lines <= 0)
return;
length += lines;
ensureCapacity(length);
int len = index + lines;
System.arraycopy(lineInfo, index, lineInfo, len,
lineInfo.length - len);
for (int i = index + lines - 1; i >= index; i--) {
lineInfo[i] = Token.NULL;
}
}
/**
* Informs the token marker that line have been deleted from
* the document. This removes the lines in question from the
* <code>lineInfo</code> array.
*
* @param index The first line number
* @param lines The number of lines
*/
public void deleteLines(int index, int lines) {
if (lines <= 0)
return;
int len = index + lines;
length -= lines;
System.arraycopy(lineInfo, len, lineInfo,
index, lineInfo.length - len);
}
/**
* Returns the number of lines in this token marker.
*/
public int getLineCount() {
return length;
}
/**
* Returns true if the next line should be repainted. This
* will return true after a line has been tokenized that starts
* a multiline token that continues onto the next line.
*/
public boolean isNextLineRequested() {
return nextLineRequested;
}
/**
* Ensures that the <code>lineInfo</code> array can contain the
* specified index. This enlarges it if necessary. No action is
* taken if the array is large enough already.<p>
* <p>
* It should be unnecessary to call this under normal
* circumstances; <code>insertLine()</code> should take care of
* enlarging the line info array automatically.
*
* @param index The array index
*/
protected void ensureCapacity(int index) {
if (lineInfo == null) {
lineInfo = new byte[index + 1];
} else if (lineInfo.length <= index) {
byte[] lineInfoN = new byte[(index + 1) * 2];
System.arraycopy(lineInfo, 0, lineInfoN, 0,
lineInfo.length);
lineInfo = lineInfoN;
}
}
/**
* Adds a token to the token list.
*
* @param length The length of the token
* @param id The id of the token
*/
protected void addToken(int length, byte id) {
if (id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST) {
throw new InternalError("Invalid id: " + id);
}
if (length == 0 && id != Token.END) {
return;
}
if (firstToken == null) {
firstToken = new Token(length, id);
lastToken = firstToken;
} else if (lastToken == null) {
lastToken = firstToken;
firstToken.length = length;
firstToken.id = id;
} else if (lastToken.next == null) {
lastToken.next = new Token(length, id);
lastToken = lastToken.next;
} else {
lastToken = lastToken.next;
lastToken.length = length;
lastToken.id = id;
}
}
}