Skip to content

Commit 905682c

Browse files
wrd233wrd233
authored
Add files via upload
1 parent 6eefc86 commit 905682c

18 files changed

Lines changed: 5407 additions & 9 deletions

‎main.go‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,40 @@
11
package main
22

33
import (
4-
"fmt"
4+
. "miniSQL/src/Interpreter/lexer"
55
"strings"
6-
7-
"miniSQL/src/parser"
86
)
97

8+
// // 解析的结果
9+
// type LexerResult struct {
10+
// Token int
11+
// Literal string
12+
// }
13+
1014
func main() {
1115
s := strings.NewReader("creat table tabel_name where key <= 1")
12-
l := parser.NewScanner(s)
16+
l := NewScanner(s)
17+
p := NewTokenizer()
18+
result := &LexerResult{}
19+
lastToken := 0
1320
for {
1421
tok, str := l.Scan()
15-
fmt.Print(tok)
16-
fmt.Print(" ")
17-
fmt.Println(str)
18-
if tok == 1 {
19-
break
22+
// fmt.Print(tok)
23+
// fmt.Print(" ")
24+
// fmt.Println(str)
25+
// if tok == 1 {
26+
// break
27+
// }
28+
switch tok {
29+
case T_EOF:
30+
// Stop lex
31+
case T_IDENT, T_INTEGER, T_FLOAT, T_STRING, T_LEFT_PARENTHESIS, T_RIGHT_PARENTHESIS, T_COMMA, T_SEMICOLON, T_EQUAL, T_ANGLE_LEFT, T_ANGLE_RIGHT, T_ANGLE_LEFT_EQUAL, T_ANGLE_RIGHT_EQUAL, T_NOT_EQUAL, T_ASTERISK, T_POINT:
32+
result.Literal = str
33+
// default:
34+
// log.Printf("UnexpectedToken: tok is %d, lit is %s\n", tok, lit)
35+
// return nil, UnexpectedTokenErr
2036
}
37+
38+
result.Token = p.FromStrLit(str, tok, lastToken)
2139
}
2240
}

‎src/Interpreter/lexer/lexer.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package lexer
2+
3+
import (
4+
"io"
5+
"log"
6+
)
7+
8+
type Token int
9+
10+
// 工具类：接受由Scanner传入的经过基本处理的token，通过lookahead来赋予该token更多信息
11+
type Tokenizer interface {
12+
FromStrLit(lit string, TokenType Token, lastToken int) int
13+
}
14+
15+
// 工具类：初步处理输入，得到基础的token
16+
type Scanner interface {
17+
Scan() (tok Token, lit string)
18+
}
19+
20+
// 模块类：将输入解析为token的总体工具类
21+
type LexerImpl struct {
22+
scanner Scanner // 处理输入的工具类
23+
tokenizer Tokenizer // 进一步赋予token信息的工具类
24+
Result interface{}
25+
}
26+
27+
// 存储类：存储token解析的结果，最终的Lex()主要利用这个对象来返回结果
28+
type LexerResult struct {
29+
Token int
30+
Literal string
31+
}
32+
33+
// 新建一个“将输入解析为token”的总体工具类的实例化对象
34+
func NewLexerImpl(r io.Reader) *LexerImpl {
35+
return &LexerImpl{
36+
scanner: NewScanner(r),
37+
tokenizer: NewTokenizer(),
38+
}
39+
}
40+
41+
func (li *LexerImpl) Lex(lastToken int) (*LexerResult, error) {
42+
result := &LexerResult{}
43+
44+
tok, lit := li.scanner.Scan() // 这里的scanner已经完成了输入流的基础token化
45+
46+
switch tok {
47+
case T_EOF:
48+
// Stop lex
49+
case T_IDENT, T_INTEGER, T_FLOAT, T_STRING, T_LEFT_PARENTHESIS, T_RIGHT_PARENTHESIS, T_COMMA, T_SEMICOLON, T_EQUAL, T_ANGLE_LEFT, T_ANGLE_RIGHT, T_ANGLE_LEFT_EQUAL, T_ANGLE_RIGHT_EQUAL, T_NOT_EQUAL, T_ASTERISK, T_POINT:
50+
result.Literal = lit
51+
default:
52+
log.Printf("UnexpectedToken: tok is %d, lit is %s\n", tok, lit)
53+
// return nil, UnexpectedTokenErr // ToDo:这里的错误机制需要完善
54+
return nil, nil
55+
}
56+
57+
result.Token = li.tokenizer.FromStrLit(lit, tok, lastToken)
58+
59+
return result, nil
60+
}

‎src/Interpreter/lexer/lexer_test.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package lexer
2+
3+
import (
4+
"log"
5+
"strings"
6+
"testing"
7+
)
8+
9+
var sql_strings = []string{
10+
// "create table 1_a",
11+
" ",
12+
"create table cxz(" +
13+
"afsdfsad int unique," +
14+
"what char(30) not null," +
15+
"primary key (what)" +
16+
");",
17+
"select a,b,c,d,e,f,g from cxz where a=123 and b=456 or c=234;",
18+
}
19+
20+
//DoTo:这里有个bug就是我们无法正确地停止输入
21+
func TestLexerLex(t *testing.T) {
22+
for _, str := range sql_strings {
23+
LastToken := 0
24+
io := strings.NewReader(str) // 组装io
25+
impl := NewLexerImpl(io) // 组装待测试的LexerImpl
26+
for LastToken != int(T_EOF) {
27+
r, _ := impl.Lex(LastToken)
28+
29+
tokVal := r.Token
30+
literal := r.Literal
31+
LastToken = tokVal
32+
33+
// log.Print(tokVal)
34+
// log.Print(" ")
35+
log.Print(literal)
36+
// log.Print(" ")
37+
// log.Print(LastToken)
38+
if tokVal == 0 {
39+
// 检测是否到达输入末尾
40+
break
41+
}
42+
}
43+
44+
}
45+
t.Errorf("不要慌，我只是想要一个log而已\n")
46+
}

‎src/Interpreter/lexer/scanner.go‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
package lexer
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"io"
7+
)
8+
9+
// type Token int
10+
11+
// 词法分析中的初步结果(部分内容可以经过tokenize来形成语义更加明确的token)
12+
const (
13+
// 特殊标记
14+
T_ILLEGAL Token = iota
15+
T_EOF
16+
WS // 空白字符
17+
// 常规类型数据
18+
T_IDENT // ID，此时我们并不区分关键词，而是归类到同一类
19+
T_INTEGER // 整数
20+
T_FLOAT // 浮点数
21+
T_STRING // 字符串
22+
// 其他标记
23+
T_ASTERISK // *
24+
T_COMMA // ,
25+
T_LEFT_PARENTHESIS // (
26+
T_RIGHT_PARENTHESIS // )
27+
T_SEMICOLON // ;
28+
T_EQUAL // =
29+
T_ANGLE_LEFT // <
30+
T_ANGLE_LEFT_EQUAL //<=
31+
T_ANGLE_RIGHT_EQUAL //>=
32+
T_ANGLE_RIGHT // >
33+
T_NOT_EQUAL // <> or !=
34+
T_POINT // .
35+
)
36+
37+
type State int // 状态机的状态
38+
39+
const (
40+
STATE_INIT State = iota
41+
STATE_INTEGER
42+
STATE_POINT
43+
STATE_FRACTION
44+
STATE_IDENT
45+
STATE_ANGLE_LEFT
46+
STATE_ANGLE_RIGHT
47+
STATE_END
48+
)
49+
50+
type CharType int // 单个字符的数据类型
51+
52+
const (
53+
NUM CharType = iota
54+
CHAR
55+
SPECIAL_SYMBOL
56+
ILLEGAL_SYMBOL
57+
SPACE
58+
UNDERLINE
59+
)
60+
61+
// eof represents a marker rune for the end of the reader.
62+
var eof = rune(0)
63+
64+
type InputScanner struct {
65+
r *bufio.Reader
66+
apostropne bool // apostropne is true means
67+
}
68+
69+
func NewScanner(r io.Reader) *InputScanner {
70+
return &InputScanner{r: bufio.NewReader(r), apostropne: false}
71+
}
72+
73+
// scanner不断从输入流中读取数据，尝试拼接出一个个初步解析的token
74+
func (s *InputScanner) Scan() (tok Token, lit string) {
75+
ch := s.read()
76+
var buf bytes.Buffer
77+
state := STATE_INIT
78+
for state != STATE_END {
79+
if checkCharType(ch) == ILLEGAL_SYMBOL {
80+
return T_ILLEGAL, string(ch)
81+
}
82+
// buf.WriteRune(ch)
83+
switch state {
84+
case STATE_INIT:
85+
switch checkCharType(ch) {
86+
case NUM:
87+
buf.WriteRune(ch)
88+
state = STATE_INTEGER
89+
case CHAR:
90+
buf.WriteRune(ch)
91+
state = STATE_IDENT
92+
case SPECIAL_SYMBOL:
93+
switch ch {
94+
case eof:
95+
return T_EOF, ""
96+
case '.':
97+
return T_POINT, string(ch)
98+
case '*':
99+
return T_ASTERISK, string(ch)
100+
case ',':
101+
return T_COMMA, string(ch)
102+
case '(':
103+
return T_LEFT_PARENTHESIS, string(ch)
104+
case ')':
105+
return T_RIGHT_PARENTHESIS, string(ch)
106+
case ';':
107+
return T_SEMICOLON, string(ch)
108+
case '=':
109+
return T_EQUAL, string(ch)
110+
case '<':
111+
buf.WriteRune(ch)
112+
state = STATE_ANGLE_LEFT
113+
case '>':
114+
buf.WriteRune(ch)
115+
state = STATE_ANGLE_RIGHT
116+
}
117+
case SPACE:
118+
case UNDERLINE:
119+
return T_ILLEGAL, string(ch)
120+
}
121+
case STATE_INTEGER:
122+
switch checkCharType(ch) {
123+
case NUM:
124+
buf.WriteRune(ch)
125+
case CHAR, SPACE, UNDERLINE:
126+
s.unread()
127+
return T_INTEGER, buf.String()
128+
case SPECIAL_SYMBOL:
129+
if ch == '.' {
130+
buf.WriteRune(ch)
131+
state = STATE_POINT
132+
} else {
133+
s.unread()
134+
return T_INTEGER, buf.String()
135+
}
136+
}
137+
case STATE_POINT:
138+
switch checkCharType(ch) {
139+
case NUM:
140+
buf.WriteRune(ch)
141+
state = STATE_FRACTION
142+
case CHAR, SPECIAL_SYMBOL, SPACE, UNDERLINE:
143+
return T_ILLEGAL, string(ch)
144+
}
145+
case STATE_FRACTION:
146+
switch checkCharType(ch) {
147+
case NUM:
148+
buf.WriteRune(ch)
149+
case CHAR, SPECIAL_SYMBOL, SPACE, UNDERLINE:
150+
s.unread()
151+
return T_FLOAT, buf.String()
152+
}
153+
case STATE_IDENT:
154+
switch checkCharType(ch) {
155+
case NUM, CHAR, UNDERLINE:
156+
buf.WriteRune(ch)
157+
case SPECIAL_SYMBOL, SPACE:
158+
s.unread()
159+
return T_IDENT, buf.String()
160+
}
161+
case STATE_ANGLE_LEFT:
162+
switch checkCharType(ch) {
163+
case NUM, CHAR, SPACE:
164+
s.unread()
165+
return T_ANGLE_LEFT, buf.String()
166+
case SPECIAL_SYMBOL:
167+
// ch = s.read()
168+
if ch == '=' {
169+
return T_ANGLE_LEFT_EQUAL, "<="
170+
} else if ch == '>' {
171+
return T_NOT_EQUAL, "<>"
172+
} else {
173+
s.unread()
174+
return T_ANGLE_LEFT, buf.String()
175+
}
176+
}
177+
case STATE_ANGLE_RIGHT:
178+
switch checkCharType(ch) {
179+
case NUM, CHAR, SPACE:
180+
s.unread()
181+
return T_ANGLE_RIGHT, buf.String()
182+
case SPECIAL_SYMBOL:
183+
// ch = s.read()
184+
if ch == '=' {
185+
return T_ANGLE_RIGHT_EQUAL, ">="
186+
} else {
187+
s.unread()
188+
return T_ANGLE_RIGHT, buf.String()
189+
}
190+
}
191+
}
192+
ch = s.read()
193+
}
194+
195+
return T_ILLEGAL, string(ch)
196+
}
197+
198+
// read reads the next rune from the buffered reader.
199+
// Returns the rune(0) if an error occurs (or io.T_EOF is returned).
200+
func (s *InputScanner) read() rune {
201+
ch, _, err := s.r.ReadRune()
202+
if err != nil {
203+
return eof
204+
}
205+
return ch
206+
}
207+
208+
// unread places the previously read rune back on the reader.
209+
func (s *InputScanner) unread() { _ = s.r.UnreadRune() }
210+
211+
func checkCharType(ch rune) CharType {
212+
if ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' {
213+
return CHAR
214+
} else if ch >= '0' && ch <= '9' {
215+
// fmt.Println("检测到数字")
216+
return NUM
217+
} else if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' {
218+
return SPACE
219+
} else if ch == '.' || ch == '*' || ch == ',' || ch == '(' || ch == ')' || ch == ';' || ch == '=' || ch == '<' || ch == '>' || ch == eof {
220+
return SPECIAL_SYMBOL
221+
} else if ch == '_' {
222+
return UNDERLINE
223+
} else {
224+
return ILLEGAL_SYMBOL
225+
}
226+
}

‎src/Interpreter/lexer/scanner_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package lexer
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func TestScan(t *testing.T) {
8+
// ToDo:现在我是懒得写了
9+
}

0 commit comments

Comments
 (0)