Skip to content

Commit

Permalink
fix:(decoder) not clear memory after decoding failed (#346)
Browse files Browse the repository at this point in the history 
* fix: not clear memory after decoding failed

* doc: update sonic ast benchmark data
  • Loading branch information
@AsterDY
AsterDY authored Jan 10, 2023
1 parent 2dc405d commit 32877b6
Show file tree
Hide file tree
Showing 7 changed files with 6 additions and 101 deletions.
12 changes: 6 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,23 +62,23 @@ BenchmarkDecoder_Parallel_Binding_StdLib-16 27582 ns/op 472.5
BenchmarkDecoder_Parallel_Binding_JsonIter-16 13571 ns/op 960.51 MB/s 14685 B/op 385 allocs/op
BenchmarkDecoder_Parallel_Binding_GoJson-16 10031 ns/op 1299.51 MB/s 22111 B/op 49 allocs/op
BenchmarkGetOne_Sonic-16 11650 ns/op 1117.81 MB/s 29 B/op 1 allocs/op
BenchmarkGetOne_Sonic-16 3276 ns/op 3975.78 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Gjson-16 9431 ns/op 1380.81 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Jsoniter-16 51178 ns/op 254.46 MB/s 27936 B/op 647 allocs/op
BenchmarkGetOne_Parallel_Sonic-16 1955 ns/op 6659.94 MB/s 125 B/op 1 allocs/op
BenchmarkGetOne_Parallel_Sonic-16 216.7 ns/op 60098.95 MB/s 24 B/op 1 allocs/op
BenchmarkGetOne_Parallel_Gjson-16 1076 ns/op 12098.62 MB/s 0 B/op 0 allocs/op
BenchmarkGetOne_Parallel_Jsoniter-16 17741 ns/op 734.06 MB/s 27945 B/op 647 allocs/op
BenchmarkSetOne_Sonic-16 16124 ns/op 807.70 MB/s 1787 B/op 17 allocs/op
BenchmarkSetOne_Sonic-16 9571 ns/op 1360.61 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Sjson-16 36456 ns/op 357.22 MB/s 52180 B/op 9 allocs/op
BenchmarkSetOne_Jsoniter-16 79475 ns/op 163.86 MB/s 45862 B/op 964 allocs/op
BenchmarkSetOne_Parallel_Sonic-16 2383 ns/op 5465.02 MB/s 2186 B/op 17 allocs/op
BenchmarkSetOne_Parallel_Sonic-16 850.9 ns/op 15305.31 MB/s 1584 B/op 17 allocs/op
BenchmarkSetOne_Parallel_Sjson-16 18194 ns/op 715.77 MB/s 52247 B/op 9 allocs/op
BenchmarkSetOne_Parallel_Jsoniter-16 33560 ns/op 388.05 MB/s 45892 B/op 964 allocs/op
```
- [Small](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 keys, 3 layers)
![small benchmarks](bench-small.jpg)
![small benchmarks](bench-small.png)
- [Large](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635KB, 10000+ key, 6 layers)
![large benchmarks](bench-large.jpg)
![large benchmarks](bench-large.png)

See [bench.sh](https://github.com/bytedance/sonic/blob/main/bench.sh) for benchmark codes.

Expand Down
Binary file removed bench-large.jpg
View file
Binary file not shown.
Binary file added bench-large.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file removed bench-small.jpg
View file
Binary file not shown.
Binary file added bench-small.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
15 changes: 0 additions & 15 deletions decoder/decoder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
`encoding/json`
`reflect`
`runtime`
`unsafe`

`github.com/bytedance/sonic/internal/native`
`github.com/bytedance/sonic/internal/native/types`
Expand Down Expand Up @@ -96,25 +95,11 @@ func (self *Decoder) Decode(val interface{}) error {
if vp == nil || vv.Type.Kind() != reflect.Ptr {
return &json.InvalidUnmarshalError{Type: vv.Type.Pack()}
}
initalized := (vv.Type.Pack().Elem().Kind() == reflect.Ptr) && (*(*unsafe.Pointer)(vp) != nil)

/* create a new stack, and call the decoder */
sb, etp := newStack(), rt.PtrElem(vv.Type)
nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)

if err != nil {
// clear val memory when decode failed,
// not including MismatcheTypeError
if _, ok := err.(*MismatchTypeError); !ok {
ev := reflect.ValueOf(val).Elem()
if initalized {
ev.Elem().Set(reflect.Zero(ev.Elem().Type()))
} else {
ev.Set(reflect.Zero(ev.Type()))
}
}
}

/* return the stack back */
self.i = nb
freeStack(sb)
Expand Down
80 changes: 0 additions & 80 deletions decoder/decoder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,86 +169,6 @@ func TestSkipMismatchTypeError(t *testing.T) {
})
}

type testStruct struct {
A int `json:"a"`
B string `json:"b"`
}

func TestClearMemWhenError(t *testing.T) {
var data = `{"a":1,"b":"1"]`
var v, v2 testStruct
_, err := decode(data, &v, false)
err2 := json.Unmarshal([]byte(data), &v2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, v2, v)

var z, z2 = new(testStruct), new(testStruct)
_, err = decode(data, z, false)
err2 = json.Unmarshal([]byte(data), z2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, z2, z)

var y, y2 *testStruct
_, err = decode(data, &y, false)
err2 = json.Unmarshal([]byte(data), &y2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, y2, y)

var x, x2 = new(testStruct), new(testStruct)
_, err = decode(data, &x, false)
err2 = json.Unmarshal([]byte(data), &x2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, x2, x)

var a, a2 interface{}
_, err = decode(data, &a, false)
err2 = json.Unmarshal([]byte(data), &a2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, a2, a)

var b, b2 = new(interface{}), new(interface{})
_, err = decode(data, b, false)
err2 = json.Unmarshal([]byte(data), b2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, b2, b)

var c, c2 *interface{}
_, err = decode(data, &c, false)
err2 = json.Unmarshal([]byte(data), &c2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, c2, c)

var d, d2 = new(interface{}), new(interface{})
_, err = decode(data, &d, false)
err2 = json.Unmarshal([]byte(data), &d2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, d2, d)

var e, e2 map[string]interface{}
_, err = decode(data, &e, false)
err2 = json.Unmarshal([]byte(data), &e2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, e2, e)

var f, f2 = new(map[string]interface{}), new(map[string]interface{})
_, err = decode(data, &f, false)
err2 = json.Unmarshal([]byte(data), &f2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, f2, f)

var g, g2 = new(map[string]interface{}), new(map[string]interface{})
_, err = decode(data, g, false)
err2 = json.Unmarshal([]byte(data), g2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, g2, g)

var h, h2 *map[string]interface{}
_, err = decode(data, &h, false)
err2 = json.Unmarshal([]byte(data), &h2)
assert.Equal(t, err2 == nil, err == nil)
assert.Equal(t, h2, h)
}

func TestDecodeCorrupt(t *testing.T) {
var ds = []string{
`{,}`,
Expand Down

0 comments on commit 32877b6

Please sign in to comment.