-
Notifications
You must be signed in to change notification settings - Fork 0
/
tokenizer.go
76 lines (69 loc) · 1.38 KB
/
tokenizer.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
package main
type token struct {
kind string
literal string
}
const (
_ int = iota
LOWEST
EQUALS // ==
LESSGREATER // > or <
SUM // +
PRODUCT // *
PREFIX // -X or !X
CALL // myFunction(X)
)
var (
idx = 0
tokens = []token{}
)
// naive tokenizer just get job done
func getTokens(input string) {
idx = 0
tokens = []token{}
for i := 0; i < len(input); i++ {
ch := input[i]
if isWhitespace(ch) {
continue
}
if isDigit(ch) {
number := string(ch)
i++
if i == len(input) {
tokens = append(tokens, token{
kind: "NUMBER", literal: number,
})
break
}
for isDigit(input[i]) {
number += string(input[i])
i++
}
i--
tokens = append(tokens, token{
kind: "NUMBER", literal: number,
})
}
if ch == '+' {
tokens = append(tokens, token{kind: "PLUS", literal: "+"})
}
if ch == '*' {
tokens = append(tokens, token{kind: "ASTERISK", literal: "*"})
}
if ch == '(' {
tokens = append(tokens, token{kind: "LPAREN", literal: "("})
}
if ch == ')' {
tokens = append(tokens, token{kind: "RPAREN", literal: ")"})
}
}
}
// func isLetter(ch byte) bool {
// return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
// }
func isDigit(ch byte) bool {
return '0' <= ch && ch <= '9'
}
func isWhitespace(ch byte) bool {
return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r'
}