Skip to content

Commit e523a50

Browse files
liusongweiliusongwei
committedMar 9, 2013
constants and formats
1 parent 42c5677 commit e523a50

12 files changed

+162
-119
lines changed
 

‎text/README.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# 包名
22

3-
函数列表
3+
包列表
44

5-
- xxx1
6-
- xxx2
5+
- [text/scanner](scanner)
6+
- xxx2

‎text/scanner/Constants.md

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
const (
2+
ScanIdents = 1 << -Ident
3+
ScanInts = 1 << -Int
4+
ScanFloats = 1 << -Float // includes Ints
5+
ScanChars = 1 << -Char
6+
ScanStrings = 1 << -String
7+
ScanRawStrings = 1 << -RawString
8+
ScanComments = 1 << -Comment
9+
SkipComments = 1 << -skipComment // if set with ScanComments, comments become white space
10+
GoTokens = ScanIdents | ScanFloats | ScanChars | ScanStrings | ScanRawStrings | ScanComments | SkipComments
11+
)
12+
13+
用于控制词法单元识别的预定义模式位。比如，通过以下模式位设置，可以配置一个只识别（Go）标识符、整型、并且忽略注释的词法解析器：
14+
15+
ScanIdents | ScanInts | SkipComments
16+
17+
const (
18+
EOF = -(iota + 1)
19+
Ident
20+
Int
21+
Float
22+
Char
23+
String
24+
RawString
25+
Comment
26+
)
27+
28+
通过Scan方法可以获取的词法单元，此外还可以获取Unicode字符。
29+
30+
const GoWhitespace = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' '
31+
32+
GoWhitespace，Scanner的空白字符的默认值。
33+

‎text/scanner/Init.md

+16-16
Original file line numberDiff line numberDiff line change
@@ -14,28 +14,28 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int hello = 3; hello+23; print hello;")
27+
src := strings.NewReader("int hello = 3; hello+23; print hello;")
2828

29-
fmt.Println(src)
29+
fmt.Println(src)
3030

31-
var s scanner.Scanner
32-
s.Init(src)
31+
var s scanner.Scanner
32+
s.Init(src)
3333

34-
tok := s.Scan()
35-
fmt.Println(s.TokenText())
36-
for tok != scanner.EOF{
37-
tok = s.Scan()
34+
tok := s.Scan()
3835
fmt.Println(s.TokenText())
36+
for tok != scanner.EOF{
37+
tok = s.Scan()
38+
fmt.Println(s.TokenText())
39+
}
3940
}
40-
}
4141

‎text/scanner/IsValid.md

+14-14
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,25 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int num = 1;")
28-
var s scanner.Scanner
27+
src := strings.NewReader("int num = 1;")
28+
var s scanner.Scanner
2929

30-
s.Init(src)
30+
s.Init(src)
3131

32-
s.Next()
32+
s.Next()
3333

34-
pos := s.Pos()
35-
fmt.Println(pos.IsValid())
34+
pos := s.Pos()
35+
fmt.Println(pos.IsValid())
3636

37-
}
37+
}
3838

‎text/scanner/Next.md

+13-13
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,24 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import (
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import (
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main() {
25+
func main() {
2626

27-
src := strings.NewReader("int num = 1;")
27+
src := strings.NewReader("int num = 1;")
2828

29-
var s scanner.Scanner
29+
var s scanner.Scanner
3030

31-
s.Init(src)
31+
s.Init(src)
3232

33-
//this will print "i" to stdout
34-
fmt.Println(scanner.TokenString(s.Next()))
33+
//this will print "i" to stdout
34+
fmt.Println(scanner.TokenString(s.Next()))
3535

36-
}
36+
}
3737

‎text/scanner/Peek.md

+13-13
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,22 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int num = 1;")
27+
src := strings.NewReader("int num = 1;")
2828

29-
var s scanner.Scanner
30-
s.Init(src)
29+
var s scanner.Scanner
30+
s.Init(src)
3131

32-
//this will print "i" to stdout
33-
fmt.Println(scanner.TokenString(s.Peek()))
34-
}
32+
//this will print "i" to stdout
33+
fmt.Println(scanner.TokenString(s.Peek()))
34+
}
3535

‎text/scanner/Position.md

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
type Position struct {
2+
Filename string // 文件名，可能为空
3+
Offset int // 位移，从0计
4+
Line int // 行数，从1计
5+
Column int // 列数，从1计（按字符计算）
6+
}
7+
8+
Position用来表示源中的位置。行数Line > 0的情况下，Position合法。
9+

‎text/scanner/README.md

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
# text/scanner包函数列表
22

3-
Constants
3+
- [Constants](Constants.md)
44
- [func TokenString(tok rune) string](TokenString.md)
5-
type Position
5+
- [type Position](Position.md)
66
- [func (pos *Position) IsValid() bool](IsValid.md)
77
- [func (pos Position) String() string](String.md)
8-
type Scanner
8+
- type Scanner
9+
Scanner用于从io.Reader中读取Unicode字符或词法单元
910
- [func (s *Scanner) Init(src io.Reader) *Scanner](Init.md)
1011
- [func (s *Scanner) Next() rune](Next.md)
1112
- [func (s *Scanner) Peek() rune](Peek.md)

‎text/scanner/Scan.md

+14-14
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,23 @@ Scan方法读取源中下一个token或字符，并返回。如果读到源的
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int num = 1;")
28-
var s scanner.Scanner
27+
src := strings.NewReader("int num = 1;")
28+
var s scanner.Scanner
2929

30-
s.Init(src)
30+
s.Init(src)
3131

32-
s.Scan()
33-
//this will print the next token "int "to stdout
34-
fmt.Println(s.TokenText())
35-
}
32+
s.Scan()
33+
//this will print the next token "int "to stdout
34+
fmt.Println(s.TokenText())
35+
}
3636

‎text/scanner/String.md

+17-17
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,27 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int num = 1;")
28-
var s scanner.Scanner
27+
src := strings.NewReader("int num = 1;")
28+
var s scanner.Scanner
2929

30-
s.Init(src)
30+
s.Init(src)
3131

32-
s.Scan()
33-
//this will print "1:4" to stdout
34-
fmt.Println(s.Pos().String())
32+
s.Scan()
33+
//this will print "1:4" to stdout
34+
fmt.Println(s.Pos().String())
3535

36-
s.Next()
37-
//this will print "1:5" to stdout
38-
fmt.Println(s.Pos().String())
39-
}
36+
s.Next()
37+
//this will print "1:5" to stdout
38+
fmt.Println(s.Pos().String())
39+
}
4040

‎text/scanner/TokenString.md

+10-10
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,18 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"text/scanner"
22-
)
23-
func main(){
19+
import(
20+
"fmt"
21+
"text/scanner"
22+
)
23+
func main(){
2424

25-
toks := []rune("abcdef")
25+
toks := []rune("abcdef")
2626

27-
for _, tok := range toks{
28-
fmt.Print(scanner.TokenString(tok))
27+
for _, tok := range toks{
28+
fmt.Print(scanner.TokenString(tok))
29+
}
2930
}
30-
}
3131

‎text/scanner/TokenText.md

+16-16
Original file line numberDiff line numberDiff line change
@@ -14,29 +14,29 @@
1414

1515
代码实例：
1616

17-
package main
17+
package main
1818

19-
import(
20-
"fmt"
21-
"strings"
22-
"text/scanner"
23-
)
19+
import(
20+
"fmt"
21+
"strings"
22+
"text/scanner"
23+
)
2424

25-
func main(){
25+
func main(){
2626

27-
src := strings.NewReader("int num = 1;")
28-
var s scanner.Scanner
27+
src := strings.NewReader("int num = 1;")
28+
var s scanner.Scanner
2929

30-
s.Init(src)
30+
s.Init(src)
3131

32-
//this will print nothing
33-
fmt.Print(s.TokenText())
32+
//this will print nothing
33+
fmt.Print(s.TokenText())
3434

35-
s.Scan()
35+
s.Scan()
3636

37-
//this will print "int" to stdout
38-
fmt.Print(s.TokenText())
37+
//this will print "int" to stdout
38+
fmt.Print(s.TokenText())
3939

40-
}
40+
}
4141

4242

0 commit comments

Comments
 (0)
Please sign in to comment.