-
Notifications
You must be signed in to change notification settings - Fork 1
/
zawgyi_detector_test.go
71 lines (63 loc) · 2.18 KB
/
zawgyi_detector_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
package myanmartools
import (
"bufio"
"bytes"
"github.com/ayeminag/go-myanmartools/internal/resources"
"math"
"math/big"
"strings"
"testing"
)
const float64EqualityThreshold = 1E-9
func almostEqual(a, b float64) bool {
if b == math.Inf(-1) && a == math.Inf(-1) {
return true
}
return math.Abs(a-b) <= float64EqualityThreshold
}
func TestZawgyiDetectorBehavior(t *testing.T) {
t.Run("should produce expected results on strings with strong signals", func(t *testing.T) {
detector := NewZawgyiDetector()
strongUnicode := "အပြည်ပြည်ဆိုင်ရာ လူ့အခွင့်အရေး ကြေညာစာတမ်း"
strongZawgyi := "အျပည္ျပည္ဆိုင္ရာ လူ႔အခြင့္အေရး ေၾကညာစာတမ္း"
up := detector.GetZawgyiProbability(strongUnicode)
if up >= 0.001 {
t.Errorf("Expected strong unicode text probability to be less then 0.001, gotten %f", up)
}
zp := detector.GetZawgyiProbability(strongZawgyi)
if zp < 0.999 {
t.Errorf("Expected strong zawgyi text probability to be greater then 0.999, gotten %f", up)
}
})
}
func TestCompatibility(t *testing.T) {
t.Run("should produce the same resuls as in compatibility.csv", func(t *testing.T) {
detector := NewZawgyiDetector()
data, err := resources.Asset("resources/compatibility.tsv")
if err != nil {
t.Fatalf("could not load resources/compatibility.tsv file %#v", err)
}
buf := bytes.NewBuffer(data)
scanner := bufio.NewScanner(buf)
for scanner.Scan() {
line := scanner.Text()
entries := strings.Split(line, "\t")
floatString := entries[0]
var expectedProbability float64
if floatString == "-Infinity" {
expectedProbability = math.Inf(-1)
} else {
bigFloat, _, err := big.ParseFloat(floatString, 10, 30, big.ToNearestEven)
expectedProbability, _ = bigFloat.Float64()
if err != nil {
t.Errorf("Error while parsing floatString: %#v", err)
}
}
testInput := entries[1]
result := detector.GetZawgyiProbability(testInput)
if !almostEqual(result, expectedProbability) {
t.Errorf("Expected probability to be %.10f gotten %.10f", expectedProbability, result)
}
}
})
}