diff --git a/json/alloc_test.go b/json/alloc_test.go index d25f1f8..6500f3a 100644 --- a/json/alloc_test.go +++ b/json/alloc_test.go @@ -24,8 +24,9 @@ import ( ) func TestNoAllocsOnAverage(t *testing.T) { + seed := time.Now().UnixNano() num := 100 - r := rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(seed)) js := randJsons(r, num) jparser := NewJsonParser() @@ -34,14 +35,14 @@ func TestNoAllocsOnAverage(t *testing.T) { return func(t *testing.T) { t.Helper() if allocs := testing.AllocsPerRun(runsPerTest, f); allocs != 0 { - t.Errorf("got %v allocs, want 0 allocs", allocs) + t.Errorf("seed = %v, got %v allocs, want 0 allocs", seed, allocs) } } } for i := 0; i < num; i++ { t.Run(fmt.Sprintf("%d", i), checkNoAllocs(func() { if err := jparser.Init(js[i]); err != nil { - t.Fatal(err) + t.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) })) @@ -49,8 +50,9 @@ func TestNoAllocsOnAverage(t *testing.T) { } func TestNotASingleAllocAfterWarmUp(t *testing.T) { + seed := time.Now().UnixNano() num := 100 - r := rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(seed)) js := randJsons(r, num) pool := pool.New() jparser := NewJsonParser() @@ -59,7 +61,7 @@ func TestNotASingleAllocAfterWarmUp(t *testing.T) { // warm up buffer pool for i := 0; i < num; i++ { if err := jparser.Init(js[i%num]); err != nil { - t.Fatal(err) + t.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } @@ -70,14 +72,14 @@ func TestNotASingleAllocAfterWarmUp(t *testing.T) { return func(t *testing.T) { t.Helper() if allocs := testing.AllocsPerRun(runsPerTest, f); allocs != 0 { - t.Errorf("got %v allocs, want 0 allocs, pool allocs = %v", allocs, pool.Size()-originalPoolSize) + t.Errorf("seed = %v, got %v allocs, want 0 allocs, pool allocs = %v", seed, allocs, pool.Size()-originalPoolSize) } } } for i := 0; i < num; i++ { t.Run(fmt.Sprintf("%d", i), checkNoAllocs(func() { if err := jparser.Init(js[i]); err != nil { - t.Fatal(err) + t.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) })) @@ -85,15 +87,16 @@ func TestNotASingleAllocAfterWarmUp(t *testing.T) { } func BenchmarkAlloc(b *testing.B) { + seed := time.Now().UnixNano() num := 1000 - r := rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(seed)) js := randJsons(r, num) jparser := NewJsonParser() // exercise buffer pool for i := 0; i < num; i++ { if err := jparser.Init(js[i%num]); err != nil { - b.Fatal(err) + b.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } @@ -101,7 +104,7 @@ func BenchmarkAlloc(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { if err := jparser.Init(js[i%num]); err != nil { - b.Fatal(err) + b.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } diff --git a/json/json.go b/json/json.go index 07b3758..a047555 100644 --- a/json/json.go +++ b/json/json.go @@ -38,6 +38,7 @@ func scanString(buf []byte) (int, error) { continue case 'u': udigits = 0 + escaped = false continue } return 0, errScanString diff --git a/json/pool_test.go b/json/pool_test.go index b4baefb..92dfa15 100644 --- a/json/pool_test.go +++ b/json/pool_test.go @@ -23,9 +23,10 @@ import ( ) func BenchmarkPoolDefault(b *testing.B) { + seed := time.Now().UnixNano() // generate random jsons num := 1000 - r := rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(seed)) js := randJsons(r, num) // initialise pool @@ -34,7 +35,7 @@ func BenchmarkPoolDefault(b *testing.B) { // exercise buffer pool for i := 0; i < num; i++ { if err := jparser.Init(js[i%num]); err != nil { - b.Fatal(err) + b.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } @@ -42,7 +43,7 @@ func BenchmarkPoolDefault(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { if err := jparser.Init(js[i%num]); err != nil { - b.Fatal(err) + b.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } @@ -50,9 +51,10 @@ func BenchmarkPoolDefault(b *testing.B) { } func BenchmarkPoolNone(b *testing.B) { + seed := time.Now().UnixNano() // generate random jsons num := 1000 - r := rand.New(rand.NewSource(time.Now().UnixNano())) + r := rand.New(rand.NewSource(seed)) js := randJsons(r, num) // set pool to no pool @@ -63,7 +65,7 @@ func BenchmarkPoolNone(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { if err := jparser.Init(js[i%num]); err != nil { - b.Fatal(err) + b.Fatalf("seed = %v, err = %v", seed, err) } walk(jparser) } diff --git a/json/rand_test.go b/json/rand_test.go index 2c8bbcb..cda757c 100644 --- a/json/rand_test.go +++ b/json/rand_test.go @@ -15,11 +15,30 @@ package json import ( - "encoding/json" + "fmt" "math/rand" "strings" + "testing" + "time" ) +func TestParseRandom(t *testing.T) { + seed := time.Now().UnixNano() + seed = 1738425704510828000 // TODO remove this line + num := 10000 + r := rand.New(rand.NewSource(seed)) + js := randJsons(r, num) + jparser := NewJsonParser() + + // warm up buffer pool + for i := 0; i < num; i++ { + if err := jparser.Init(js[i%num]); err != nil { + t.Fatalf("seed = %v, err = %v, input = %v", seed, err, string(js[i%num])) + } + walk(jparser) + } +} + func randJsons(r *rand.Rand, num int) [][]byte { js := make([][]byte, num) for i := 0; i < num; i++ { @@ -30,68 +49,69 @@ func randJsons(r *rand.Rand, num int) [][]byte { func randJson(r *rand.Rand) []byte { val := randValue(r, 5) - data, err := json.Marshal(val) - if err != nil { - panic(err) - } - return data -} - -func randObject(r *rand.Rand, level int) map[string]interface{} { - l := r.Intn(10) - ms := make(map[string]interface{}) - for i := 0; i < l; i++ { - ms[randName(r)] = randValue(r, level) - } - return ms + return []byte(val) } -func randArray(r *rand.Rand, level int) []interface{} { - l := r.Intn(10) - as := make([]interface{}, l) - for i := 0; i < l; i++ { - as[i] = randValue(r, level) - } - return as -} - -func randValue(r *rand.Rand, level int) interface{} { - maxN := 9 +// value := object | array | string | number | "true" | "false" | "null" +func randValue(r *rand.Rand, level int) string { + maxN := 6 if level <= 0 { // do not generate arrays or objects, // since we have generated a deep enough structure and // we do not want to endlessly recurse. - maxN = 7 + maxN = 4 } switch r.Intn(maxN) { case 0: - return nil + return "null" case 1: - return bool(r.Intn(2) == 0) + return "false" case 2: - return int64(r.Int63()) + return "true" case 3: - return uint64(r.Uint64()) + return randNumber(r) case 4: - return float64(r.Float64()) - case 5: return randString(r) - case 6: - return randBytes(r) - case 7: + case 5: return randArray(r, level-1) - case 8: + case 6: return randObject(r, level-1) } panic("unreachable") } -func randBytes(r *rand.Rand) []byte { - bs := make([]byte, int(r.Intn(100))) - for i := range bs { - bs[i] = byte(r.Intn(255)) +// object := '{' ws '}' | '{' members '}' +// members := member | member ',' members +// member := ws string ws ':' element +func randObject(r *rand.Rand, level int) string { + l := r.Intn(10) + if l == 0 { + return "{" + randWs(r) + "}" + } + ss := make([]string, l) + for i := 0; i < l; i++ { + ss[i] = randWs(r) + randString(r) + randWs(r) + ":" + randElement(r, level) } - return bs + return "{" + strings.Join(ss, ",") + "}" +} + +// array := '[' ws ']' | '[' elements ']' +// elements := element | element ',' elements +func randArray(r *rand.Rand, level int) string { + l := r.Intn(10) + if l == 0 { + return "[" + randWs(r) + "]" + } + ss := make([]string, l) + for i := 0; i < l; i++ { + ss[i] = randElement(r, level) + } + return "[" + strings.Join(ss, ",") + "]" +} + +// element := ws value ws +func randElement(r *rand.Rand, level int) string { + return randWs(r) + randValue(r, level) + randWs(r) } // string := '"' characters '"' @@ -101,7 +121,7 @@ func randString(r *rand.Rand) string { for i := range ss { ss[i] = randChar(r) } - s := strings.Join(ss, "") + s := "\"" + strings.Join(ss, "") + "\"" return s } @@ -109,54 +129,148 @@ func randString(r *rand.Rand) string { func randChar(r *rand.Rand) string { switch r.Intn(2) { case 0: - max := int('\U0010FFFF') + 1 min := int('\u0020') + max := int('\U0010FFFF') + 1 ran := int((max - min) - 2) random := rune(r.Intn(ran) + min) - if random >= '"' { - random += 1 - } - if random >= '\\' { - random += 1 + if random != '"' && random != '\\' { + return string([]rune{random}) } - return string([]rune{random}) + return randChar(r) case 1: - return string([]rune{randEscape(r)}) + return "\\" + randEscape(r) } panic("unreachable") } // escape := '"' | '\' | '/' | 'b' | 'f' | 'n' | 'r' | 't' | 'u' hex hex hex hex -// hex := digit | 'A' . 'F' | 'a' . 'f' -func randEscape(r *rand.Rand) rune { +func randEscape(r *rand.Rand) string { switch r.Intn(9) { case 0: - return '"' + return "\"" case 1: - return '\\' + return "\\" case 2: - return '/' + return "/" case 3: - return '\b' + return "b" case 4: - return '\f' + return "f" case 5: - return '\n' + return "n" case 6: - return '\r' + return "r" case 7: - return '\t' + return "t" case 8: - // TODO generate better \u characters - return '\u01aB' + return "u" + randHex(r) + randHex(r) + randHex(r) + randHex(r) } panic("unreachable") } -func randName(r *rand.Rand) string { - ss := make([]byte, int(r.Intn(100))) - for i := range ss { - ss[i] = byte(65 + r.Intn(26)) +// number := integer fraction exponent +func randNumber(r *rand.Rand) string { + return randInteger(r) + randFraction(r) + randExponent(r) +} + +// integer := digit | onenine digits | '-' digit | '-' onenine digits +func randInteger(r *rand.Rand) string { + switch r.Intn(4) { + case 0: + return randDigit(r) + case 1: + return randOneNine(r) + randDigits(r) + case 2: + return "-" + randDigit(r) + case 3: + return "-" + randOneNine(r) + randDigits(r) + } + panic("unreachable") +} + +// exponent := "" | 'E' sign digits | 'e' sign digits +func randExponent(r *rand.Rand) string { + switch r.Intn(3) { + case 0: + return "" + case 1: + return "E" + randSign(r) + randDigits(r) + case 2: + return "3" + randSign(r) + randDigits(r) + } + panic("unreachable") +} + +// fraction := "" | '.' digits +func randFraction(r *rand.Rand) string { + switch r.Intn(2) { + case 0: + return "" + case 1: + return "." + randDigits(r) + } + panic("unreachable") +} + +// sign := "" | '+' | '-' +func randSign(r *rand.Rand) string { + switch r.Intn(3) { + case 0: + return "" + case 1: + return "+" + case 2: + return "-" + } + panic("unreachable") +} + +// digits := digit | digit digits +func randDigits(r *rand.Rand) string { + l := r.Intn(5) + 1 + ss := make([]string, l) + for i := 0; i < l; i++ { + ss[i] = randDigit(r) + } + return strings.Join(ss, "") +} + +// digit := '0' | onenine +func randDigit(r *rand.Rand) string { + return fmt.Sprintf("%d", r.Intn(10)) +} + +// onenine := '1' . '9' +func randOneNine(r *rand.Rand) string { + return fmt.Sprintf("%d", r.Intn(9)+1) +} + +// hex := digit | 'A' . 'F' | 'a' . 'f' +func randHex(r *rand.Rand) string { + s := "01234567890abcdefABCDEF" + return string([]rune{rune(s[r.Intn(len(s))])}) +} + +// ws := "" | '0020' ws | '000A' ws | '000D' ws | '0009' ws +func randWs(r *rand.Rand) string { + l := r.Intn(5) + ss := make([]rune, l) + for i := 0; i < l; i++ { + ss[i] = randW(r) } return string(ss) } + +func randW(r *rand.Rand) rune { + switch r.Intn(4) { + case 0: + return '\u0020' + case 1: + return '\u000A' + case 2: + return '\u000D' + case 3: + return '\u0009' + } + panic("unreachable") +}