-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathLexer.java
More file actions
96 lines (86 loc) · 2.73 KB
/
Lexer.java
File metadata and controls
96 lines (86 loc) · 2.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
package translator;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Lexer {
/**
* Ëåêñ. ðàçáîð ñòðîêè
* - íàõîäèò âñå âõîæäåíèÿ òîêåíîâ è ñîõðàíÿåò èõ â ìàññèâ
* ïî èíäåêñó ðàâíîìó èíäåêñó íà÷àëà òîêåíà â ñòðîêå
* - âàëèäàöèÿ òîêåíîâ (îòáðàñûâàíèå îøèáî÷íûõ)
* @param str
* @return List<Token>
* @throws LexException
*/
public List<Token> read(String str) throws LexException {
List<Token> tokens = new ArrayList<Token>();
Map<Integer, Token> map = new TreeMap<Integer, Token>();
Token tarr [] = new Token[str.length()];
str = str.trim(); // îòðåçàåì ïðîáåëû ïî êðàÿì ñòðîêè
if (str.isEmpty())
return tokens;
TokenType[] ttypes = TokenType.values(); // ìàññèâ òèïîâ òîêåíîâ
Matcher matcher;
for (int i = 0; i < ttypes.length; i++) {
Pattern p = Pattern.compile(ttypes[i].pattern);
matcher = p.matcher(str);
while (matcher.find()) {
//if(map.containsKey(matcher.start()))
//åñëè íàøåëñÿ òîêåí â ñòðîêå
// êëàäåì åãî â êàðòó <èíäåêñ ïåðâîãî ñèìâîëà, òîêåí>
map.put(matcher.start(), new Token(ttypes[i], matcher.group()));
// è êëàäåì åãî â ìàññèâ tarr[èíäåêñ ïåðâîãî ñèìâîëà] = òîêåí
tarr[matcher.start()] = new Token(ttypes[i], matcher.group());
}
}
validate(tarr, map); //âàëèäàöèÿ
int pos = 0;
for (Iterator<Entry<Integer, Token>> iterator = map.entrySet().iterator(); iterator.hasNext();) {
Entry<Integer, Token> e = (Entry<Integer, Token>) iterator.next();
if (pos < e.getKey()) {
String s = str.substring(pos, e.getKey());
throw new LexException(s + " is not a lexeme");
}
Token t = e.getValue();
tokens.add(t);
pos += t.value.length();
}
if (pos<str.length()){
String s = str.substring(pos);
throw new LexException(s+ " is not a lexeme");
}
return tokens;
}
/**
* çäåñü óäàëÿþòñÿ âñå ëèøíèå òîêåíû
* íàïðèìåð, â ñòðî÷êå "String bla-bla"
* íàéäåòñÿ ïî èíäåêñó " ñòðîêà â êàâû÷êàõ "String bla-bla",
* à ïî èíäåêñó S è b èìåíà ôóíêöèé String è bla-bla
* ïîýòîìó ìû ñìîòðèì êàêàÿ äëèíà ó ñòðîêè â êàâû÷êàõ,
* è åñëè ó íàñ ÷òî-òî åùå íàøëîñü íà ïðîòÿæåíèè ýòîé ñòðî÷êè,
* óäàëÿåì ýòî. Òàê äåëàåì äëÿ ëþáûõ òîêåíîâ
*
* @param tarr
* @param map
*/
private void validate(Token[] tarr, Map<Integer, Token> map) {
int pos = 0;
int len;
for (int i = 0; i < tarr.length; i++) {
if(tarr[i] != null){
if(pos > i){
tarr[i]=null;
map.remove(i);
}else{
len = tarr[i].value.length();
pos += len;
}
}
}
}
}