Skip to content

Commit

Permalink
improved performance for packer
Browse files Browse the repository at this point in the history 
fixed a packing bug for numbers between -32..31
  • Loading branch information
@khaf
khaf committed Dec 3, 2014
1 parent c0d92d3 commit c73a234
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 25 deletions.
44 changes: 35 additions & 9 deletions packer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,10 @@ func (pckr *packer) PackObject(obj interface{}) error {
case float64:
pckr.PackFloat64(obj.(float64))
return nil
case []interface{}:
return pckr.PackList(obj.([]interface{}))
case map[interface{}]interface{}:
return pckr.PackMap(obj.(map[interface{}]interface{}))
}

// check for array and map
Expand All @@ -204,7 +208,13 @@ func (pckr *packer) PackObject(obj interface{}) error {
}
return pckr.PackList(arr)
case reflect.Map:
return pckr.PackMap(obj.(map[interface{}]interface{}))
s := reflect.ValueOf(obj)
l := s.Len()
amap := make(map[interface{}]interface{}, l)
for _, i := range s.MapKeys() {
amap[i.Interface()] = s.MapIndex(i).Interface()
}
return pckr.PackMap(amap)
}

panic(fmt.Sprintf("Type `%v` not supported to pack.", reflect.TypeOf(obj)))
Expand Down Expand Up @@ -238,7 +248,7 @@ func (pckr *packer) PackALong(val int64) {
pckr.PackLong(0xd3, val)
} else {
if val >= -32 {
pckr.PackAByte(0xe0 | byte(val) + 32)
pckr.PackAByte(0xe0 | (byte(val) + 32))
return
}

Expand Down Expand Up @@ -279,7 +289,7 @@ func (pckr *packer) PackAInt(val int) {
pckr.PackInt(0xce, int32(val))
} else {
if val >= -32 {
pckr.PackAByte(0xe0 | byte(val+32))
pckr.PackAByte(0xe0 | (byte(val) + 32))
return
}

Expand All @@ -296,6 +306,16 @@ func (pckr *packer) PackAInt(val int) {
}
}

var _b8 = []byte{0, 0, 0, 0, 0, 0, 0, 0}
var _b4 = []byte{0, 0, 0, 0}
var _b2 = []byte{0, 0}

func (pckr *packer) grow(b []byte) int {
pos := pckr.buffer.Len()
pckr.buffer.Write(b)
return pos
}

func (pckr *packer) PackString(val string) {
size := len(val) + 1
pckr.PackByteArrayBegin(size)
Expand All @@ -309,22 +329,26 @@ func (pckr *packer) PackByteArray(src []byte, srcOffset int, srcLength int) {

func (pckr *packer) PackLong(valType int, val int64) {
pckr.buffer.WriteByte(byte(valType))
pckr.buffer.Write(Buffer.Int64ToBytes(val, nil, pckr.offset))
pos := pckr.grow(_b8)
pckr.buffer.Write(Buffer.Int64ToBytes(val, pckr.buffer.Bytes(), pos))
}

func (pckr *packer) PackULong(val uint64) {
pckr.buffer.WriteByte(byte(0xcf))
pckr.buffer.Write(Buffer.Int64ToBytes(int64(val), nil, pckr.offset))
pos := pckr.grow(_b8)
Buffer.Int64ToBytes(int64(val), pckr.buffer.Bytes(), pos)
}

func (pckr *packer) PackInt(valType int, val int32) {
pckr.buffer.WriteByte(byte(valType))
pckr.buffer.Write(Buffer.Int32ToBytes(val, nil, pckr.offset))
pos := pckr.grow(_b4)
Buffer.Int32ToBytes(val, pckr.buffer.Bytes(), pos)
}

func (pckr *packer) PackShort(valType int, val int16) {
pckr.buffer.WriteByte(byte(valType))
pckr.buffer.Write(Buffer.Int16ToBytes(val, nil, pckr.offset))
pos := pckr.grow(_b2)
Buffer.Int16ToBytes(val, pckr.buffer.Bytes(), pos)
}

func (pckr *packer) PackByte(valType int, val byte) {
Expand All @@ -346,12 +370,14 @@ func (pckr *packer) PackBool(val bool) {

func (pckr *packer) PackFloat32(val float32) {
pckr.buffer.WriteByte(0xca)
pckr.buffer.Write(Buffer.Float32ToBytes(val, nil, pckr.offset))
pos := pckr.grow(_b4)
Buffer.Float32ToBytes(val, pckr.buffer.Bytes(), pos)
}

func (pckr *packer) PackFloat64(val float64) {
pckr.buffer.WriteByte(0xcb)
pckr.buffer.Write(Buffer.Float64ToBytes(val, nil, pckr.offset))
pos := pckr.grow(_b8)
Buffer.Float64ToBytes(val, pckr.buffer.Bytes(), pos)
}

func (pckr *packer) PackAByte(val byte) {
Expand Down
21 changes: 15 additions & 6 deletions packing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func testPackingFor(v interface{}) interface{} {

unpacker := newUnpacker(packer.buffer.Bytes(), 0, len(packer.buffer.Bytes()))
unpackedValue, err := unpacker.unpackObject()
Expect(err).ToNot(HaveOccurred())

return unpackedValue
}
Expand All @@ -42,6 +43,14 @@ var _ = Describe("Packing Test", func() {
Expect(testPackingFor(nil)).To(BeNil())
})

It("should pack and unpack -32 < int8 < 32 values", func() {
v := int8(31)
Expect(testPackingFor(v)).To(Equal(int(v)))

v = int8(-32)
Expect(testPackingFor(v)).To(Equal(int(v)))
})

It("should pack and unpack int8 values", func() {
v := int8(math.MaxInt8)
Expect(testPackingFor(v)).To(Equal(int(v)))
Expand Down Expand Up @@ -83,10 +92,10 @@ var _ = Describe("Packing Test", func() {

It("should pack and unpack int64 values", func() {
v := int64(math.MaxInt64)
Expect(testPackingFor(v)).To(Equal(v))
Expect(testPackingFor(v)).To(Equal(int(v)))

v = int64(math.MinInt64)
Expect(testPackingFor(v)).To(Equal(v))
Expect(testPackingFor(v)).To(Equal(int(v)))
})

It("should pack and unpack uint64 values", func() {
Expand Down Expand Up @@ -273,11 +282,11 @@ var _ = Describe("Packing Test", func() {
"int32": int(math.MaxInt32),
"mint32": int(math.MinInt32),
"uint": uint64(math.MaxUint64),
"int": int64(math.MaxInt64),
"mint": int64(math.MinInt64),
"int": int(math.MaxInt64),
"mint": int(math.MinInt64),
"uint64": uint64(math.MaxUint64),
"int64": int64(math.MaxInt64),
"mint64": int64(math.MinInt64),
"int64": int(math.MaxInt64),
"mint64": int(math.MinInt64),
"maxFloat32": float32(math.MaxFloat32),
"minFloat32": float32(-math.MaxFloat32),
"maxFloat64": float64(math.MaxFloat64),
Expand Down
180 changes: 174 additions & 6 deletions udf_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"flag"
"math"
"math/rand"
"strings"
"time"

. "github.com/aerospike/aerospike-client-go"
Expand All @@ -43,6 +44,13 @@ const udfDelete = `function deleteRecord(rec)
aerospike:remove(rec) -- Delete main record, Populate the return status
end`

const udfEcho = `function echo(rec, param)
local ret = map()
ret['val'] = param
ret['str_val'] = tostring(param)
return ret -- return the same value to make sure serializations are working well
end`

// ALL tests are isolated by SetName and Key, which are 50 random charachters
var _ = Describe("UDF/Query tests", func() {
rand.Seed(time.Now().UnixNano())
Expand Down Expand Up @@ -103,11 +111,8 @@ var _ = Describe("UDF/Query tests", func() {
Expect(err).ToNot(HaveOccurred())

// wait until UDF is created
for {
if err := <-regTask.OnComplete(); err == nil {
break
}
}
err = <-regTask.OnComplete()
Expect(err).ToNot(HaveOccurred())

delTask, err := client.RemoveUDF(wpolicy, "udfToBeDropped.lua")
Expect(err).ToNot(HaveOccurred())
Expand Down Expand Up @@ -193,12 +198,175 @@ var _ = Describe("UDF/Query tests", func() {
i := 0
for fullRec := range recordset.Records {
i++
// only one recortd should be returned
// only one record should be returned
Expect(fullRec.Bins[bin1.Name]).To(Equal(math.MaxInt16 + 1))
}
Expect(i).To(Equal(1))
})

}) // context

Context("must serialize parameters and return values sensibly", func() {

regTask, _ := client.RegisterUDF(wpolicy, []byte(udfEcho), "udfEcho.lua", LUA)
// wait until UDF is created
<-regTask.OnComplete()
// a new record that is not in the range
key, err = NewKey(ns, set, randString(50))

testMatrix := map[interface{}]interface{}{
math.MinInt64: math.MinInt64,
// math.MaxInt64: int64(math.MaxInt64), // TODO: Wrong serialization on server - sign-bit is wrong
math.MinInt32: math.MinInt32, // TODO: Wrong serialization type on server
math.MaxUint32: math.MaxUint32,
math.MinInt16: math.MinInt16,
math.MaxInt16: math.MaxInt16,
math.MaxUint16: math.MaxUint16,
math.MinInt8: math.MinInt8,
math.MaxInt8: math.MaxInt8,
math.MaxUint8: math.MaxUint8,
-1: -1,
0: 0,
"": "",
strings.Repeat("s", 1): strings.Repeat("s", 1),
strings.Repeat("s", 10): strings.Repeat("s", 10),
strings.Repeat("s", 100): strings.Repeat("s", 100),
strings.Repeat("s", 1000): strings.Repeat("s", 1000),
strings.Repeat("s", 10000): strings.Repeat("s", 10000),
strings.Repeat("s", 33781): strings.Repeat("s", 33781),
strings.Repeat("s", 100000): strings.Repeat("s", 100000),
"Hello, 世界": "Hello, 世界",
}

It("must serialize nil values to echo function and get the same value back", func() {

res, err := client.Execute(nil, key, "udfEcho", "echo", NewValue(nil))
Expect(err).ToNot(HaveOccurred())
Expect(res.(map[interface{}]interface{})["val"]).To(BeNil())

}) // it

It("must serialize values to echo function and get the same value back", func() {

for k, v := range testMatrix {
res, err := client.Execute(nil, key, "udfEcho", "echo", NewValue(k))
Expect(err).ToNot(HaveOccurred())
Expect(res.(map[interface{}]interface{})["val"]).To(Equal(v))
}

}) // it

It("must serialize list values to echo function and get the same value back", func() {

v := []interface{}{
nil,
math.MinInt64,
math.MinInt32,
math.MinInt16,
math.MinInt8,
-1,
0,
1,
math.MaxInt8,
math.MaxUint8,
math.MaxInt16,
math.MaxUint16,
math.MaxInt32,
math.MaxUint32,
math.MaxInt64,
// uint64(math.MaxUint64),// TODO: Wrong serialization on server side
"",
"Hello, 世界",
}

vExpected := []interface{}{
nil,
int(math.MinInt64),
int(math.MinInt32),
int(math.MinInt16),
int(math.MinInt8),
int(-1),
int(0),
int(1),
int(math.MaxInt8),
int(math.MaxUint8),
int(math.MaxInt16),
int(math.MaxUint16),
int(math.MaxInt32),
int(math.MaxUint32),
uint64(math.MaxInt64), // TODO: Wrong serialization on server
// uint64(math.MaxUint64), // TODO: Wrong serialization on server side
"",
"Hello, 世界",
}

res, err := client.Execute(nil, key, "udfEcho", "echo", NewValue(v))

// for i := range v {
// fmt.Printf("%v => %T\n", res.(map[interface{}]interface{})["val"].([]interface{})[i], res.(map[interface{}]interface{})["val"].([]interface{})[i])
// fmt.Printf("%v => %T\n", vExpected[i], vExpected[i])
// }

Expect(err).ToNot(HaveOccurred())
Expect(res.(map[interface{}]interface{})["val"]).To(Equal(vExpected))

}) // it

It("must serialize map values to echo function and get the same value back", func() {

v := map[interface{}]interface{}{
nil: nil,
math.MinInt64: math.MinInt64,
math.MinInt32: math.MinInt32,
math.MinInt16: math.MinInt16,
math.MinInt8: math.MinInt8,
-1: -1,
0: 0,
1: 1,
math.MaxInt8: math.MaxInt8,
math.MaxUint8: math.MaxUint8,
math.MaxInt16: math.MaxInt16,
math.MaxUint16: math.MaxUint16,
math.MaxInt32: math.MaxInt32,
math.MaxUint32: math.MaxUint32,
math.MaxInt64: math.MaxInt64,
"": "",
"Hello, 世界": "Hello, 世界",
}

vExpected := map[interface{}]interface{}{
nil: nil,
math.MinInt64: math.MinInt64,
math.MinInt32: math.MinInt32,
math.MinInt16: math.MinInt16,
math.MinInt8: math.MinInt8,
-1: -1,
0: 0,
1: 1,
math.MaxInt8: math.MaxInt8,
math.MaxUint8: math.MaxUint8,
math.MaxInt16: math.MaxInt16,
math.MaxUint16: math.MaxUint16,
math.MaxInt32: math.MaxInt32,
math.MaxUint32: math.MaxUint32,
uint64(math.MaxInt64): uint64(math.MaxInt64),
"": "",
"Hello, 世界": "Hello, 世界",
}

res, err := client.Execute(nil, key, "udfEcho", "echo", NewValue(v))
Expect(err).ToNot(HaveOccurred())

resMap := res.(map[interface{}]interface{})["val"].(map[interface{}]interface{})
// for k := range resMap {
// fmt.Printf("%v : %v => %T: %T\n", k, k, resMap[k], resMap[k])
// fmt.Printf("%v => %T\n", vExpected[k], vExpected[k])
// }

Expect(resMap).To(Equal(vExpected))

}) // it

}) // context

})
5 changes: 4 additions & 1 deletion unpacker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ func (upckr *unpacker) unpackObject() (interface{}, error) {
case 0xd3:
val := Buffer.BytesToInt64(upckr.buffer, upckr.offset)
upckr.offset += 8
if Buffer.Arch64Bits {
return int(val), nil
}
return int64(val), nil

case 0xda:
Expand Down Expand Up @@ -248,7 +251,7 @@ func (upckr *unpacker) unpackObject() (interface{}, error) {
}

if theType >= 0xe0 {
return int(theType - 0xe0 - 32), nil
return int(int(theType) - 0xe0 - 32), nil
}
}

Expand Down
Loading

0 comments on commit c73a234

Please sign in to comment.