From 6ba5fc4eb7cc4e7262d37873afdcaf66a4565b8a Mon Sep 17 00:00:00 2001 From: Jay Sen Date: Thu, 15 Sep 2022 10:55:59 -0700 Subject: [PATCH 1/2] [issues-912] optimize serialization size for primitive arrays - IntOnly for now --- src/com/esotericsoftware/kryo/Kryo.java | 9 +++++++ src/com/esotericsoftware/kryo/io/Input.java | 22 +++++++++++++---- .../serializers/DefaultArraySerializers.java | 24 +++++++++++++++++-- .../kryo/serializers/ArraySerializerTest.java | 17 +++++++++++++ 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/com/esotericsoftware/kryo/Kryo.java b/src/com/esotericsoftware/kryo/Kryo.java index fa0434023..3566f3059 100644 --- a/src/com/esotericsoftware/kryo/Kryo.java +++ b/src/com/esotericsoftware/kryo/Kryo.java @@ -162,6 +162,7 @@ public class Kryo { private IdentityMap originalToCopy; private Object needsCopyReference; private Generics generics = new DefaultGenerics(this); + public boolean optimizePrimitiveArrays = false; /** Creates a new Kryo with a {@link DefaultClassResolver} and references disabled. */ public Kryo () { @@ -1294,6 +1295,14 @@ public void setOptimizedGenerics (boolean optimizedGenerics) { generics = optimizedGenerics ? new DefaultGenerics(this) : NoGenerics.INSTANCE; } + public boolean isOptimizePrimitiveArrays() { + return optimizePrimitiveArrays; + } + + public void setOptimizePrimitiveArrays(boolean optimizePrimitiveArraySize) { + this.optimizePrimitiveArrays = optimizePrimitiveArraySize; + } + static final class DefaultSerializerEntry { final Class type; final SerializerFactory serializerFactory; diff --git a/src/com/esotericsoftware/kryo/io/Input.java b/src/com/esotericsoftware/kryo/io/Input.java index e82a2b39d..81fdfaf29 100644 --- a/src/com/esotericsoftware/kryo/io/Input.java +++ b/src/com/esotericsoftware/kryo/io/Input.java @@ -19,6 +19,7 @@ package com.esotericsoftware.kryo.io; +import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.KryoBufferUnderflowException; import com.esotericsoftware.kryo.util.Pool.Poolable; @@ -904,12 +905,12 @@ private String readAscii_slow (int charCount) { // Primitive arrays: /** Reads an int array in bulk. This may be more efficient than reading them individually. */ - public int[] readInts (int length) throws KryoException { + public int[] readInts (int length, int optimizedLength) throws KryoException { int[] array = new int[length]; - if (optional(length << 2) == length << 2) { + if (optional(optimizedLength << 2) == optimizedLength << 2) { byte[] buffer = this.buffer; int p = this.position; - for (int i = 0; i < length; i++, p += 4) { + for (int i = 0; i < optimizedLength; i++, p += 4) { array[i] = buffer[p] & 0xFF // | (buffer[p + 1] & 0xFF) << 8 // | (buffer[p + 2] & 0xFF) << 16 // @@ -917,7 +918,7 @@ public int[] readInts (int length) throws KryoException { } position = p; } else { - for (int i = 0; i < length; i++) + for (int i = 0; i < optimizedLength; i++) array[i] = readInt(); } return array; @@ -932,7 +933,18 @@ public int[] readInts (int length, boolean optimizePositive) throws KryoExceptio array[i] = readVarInt(optimizePositive); return array; } - return readInts(length); + return readInts(length, length); + } + + /** same as {@link #readInts(int, boolean)} except it also takes optimizedLength when {@link Kryo#isOptimizePrimitiveArrays()} is set */ + public int[] readInts (int length, int optimizedLength, boolean optimizePositive) throws KryoException { + if (varEncoding) { + int[] array = new int[length]; + for (int i = 0; i < optimizedLength; i++) + array[i] = readVarInt(optimizePositive); + return array; + } + return readInts(length, optimizedLength); } /** Reads a long array in bulk. This may be more efficient than reading them individually. */ diff --git a/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java b/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java index a59d0a6be..ab12e1ada 100644 --- a/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java +++ b/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java @@ -71,13 +71,33 @@ public void write (Kryo kryo, Output output, int[] object) { return; } output.writeVarInt(object.length + 1, true); - output.writeInts(object, 0, object.length, false); + if(kryo.isOptimizePrimitiveArrays()){ + int optimizedSize = object.length; + while(object[optimizedSize-1]==0){ + optimizedSize--; + } +// for (int i = object.length-1; i > 0; i--) { +// if(object[i]!=0){ +// break; +// } +// optimizedSize--; +// } + output.writeVarInt(optimizedSize + 1, true); + output.writeInts(object, 0, optimizedSize, false); + }else{ + output.writeInts(object, 0, object.length, false); + } } public int[] read (Kryo kryo, Input input, Class type) { int length = input.readVarInt(true); if (length == NULL) return null; - return input.readInts(length - 1, false); + if(kryo.isOptimizePrimitiveArrays()){ + int optimizedSize = input.readVarInt(true); + return input.readInts(length-1, optimizedSize - 1, false); + }else{ + return input.readInts(length - 1, false); + } } public int[] copy (Kryo kryo, int[] original) { diff --git a/test/com/esotericsoftware/kryo/serializers/ArraySerializerTest.java b/test/com/esotericsoftware/kryo/serializers/ArraySerializerTest.java index dd0cabe2c..a318b6896 100644 --- a/test/com/esotericsoftware/kryo/serializers/ArraySerializerTest.java +++ b/test/com/esotericsoftware/kryo/serializers/ArraySerializerTest.java @@ -32,6 +32,23 @@ class ArraySerializerTest extends KryoTestCase { supportsCopy = true; } + @Test + void testOptimizedIntArrays () { + kryo.register(int[].class); + + // default case + roundTrip(6, new int[] {1, 2, 3, 4}); + // case where lot of trailing values are default value + // case withbut any optimization, it would have length of 18 + roundTrip(18, new int[] {1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + // case with optimization, it would have length of 7, and also regardless of the size now. + kryo.setOptimizePrimitiveArrays(true); + roundTrip(7, new int[] {1, 2, 3, 4, 0, 0, 0, 0, 0}); + roundTrip(7, new int[] {1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + } + @Test void testArrays () { kryo.register(int[].class); From b714f45c492e6c1b2f78a6a72956b1ecf0f8a172 Mon Sep 17 00:00:00 2001 From: Jay Sen Date: Thu, 15 Sep 2022 11:01:51 -0700 Subject: [PATCH 2/2] removing commented code --- src/com/esotericsoftware/kryo/io/Input.java | 39 ++++++++----------- .../serializers/DefaultArraySerializers.java | 6 --- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/com/esotericsoftware/kryo/io/Input.java b/src/com/esotericsoftware/kryo/io/Input.java index 81fdfaf29..0417af549 100644 --- a/src/com/esotericsoftware/kryo/io/Input.java +++ b/src/com/esotericsoftware/kryo/io/Input.java @@ -903,6 +903,22 @@ private String readAscii_slow (int charCount) { } // Primitive arrays: + public int[] readInts (int length, boolean optimizePositive) throws KryoException { + return readInts(length, length, optimizePositive); + } + + /** Reads an int array in bulk using fixed or variable length encoding, depending on + * {@link #setVariableLengthEncoding(boolean)}. This may be more efficient than reading them individually. + * It also takes optimizedLength when {@link Kryo#isOptimizePrimitiveArrays()} is set */ + public int[] readInts (int length, int optimizedLength, boolean optimizePositive) throws KryoException { + if (varEncoding) { + int[] array = new int[length]; + for (int i = 0; i < optimizedLength; i++) + array[i] = readVarInt(optimizePositive); + return array; + } + return readInts(length, optimizedLength); + } /** Reads an int array in bulk. This may be more efficient than reading them individually. */ public int[] readInts (int length, int optimizedLength) throws KryoException { @@ -924,29 +940,6 @@ public int[] readInts (int length, int optimizedLength) throws KryoException { return array; } - /** Reads an int array in bulk using fixed or variable length encoding, depending on - * {@link #setVariableLengthEncoding(boolean)}. This may be more efficient than reading them individually. */ - public int[] readInts (int length, boolean optimizePositive) throws KryoException { - if (varEncoding) { - int[] array = new int[length]; - for (int i = 0; i < length; i++) - array[i] = readVarInt(optimizePositive); - return array; - } - return readInts(length, length); - } - - /** same as {@link #readInts(int, boolean)} except it also takes optimizedLength when {@link Kryo#isOptimizePrimitiveArrays()} is set */ - public int[] readInts (int length, int optimizedLength, boolean optimizePositive) throws KryoException { - if (varEncoding) { - int[] array = new int[length]; - for (int i = 0; i < optimizedLength; i++) - array[i] = readVarInt(optimizePositive); - return array; - } - return readInts(length, optimizedLength); - } - /** Reads a long array in bulk. This may be more efficient than reading them individually. */ public long[] readLongs (int length) throws KryoException { long[] array = new long[length]; diff --git a/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java b/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java index ab12e1ada..e081e66e4 100644 --- a/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java +++ b/src/com/esotericsoftware/kryo/serializers/DefaultArraySerializers.java @@ -76,12 +76,6 @@ public void write (Kryo kryo, Output output, int[] object) { while(object[optimizedSize-1]==0){ optimizedSize--; } -// for (int i = object.length-1; i > 0; i--) { -// if(object[i]!=0){ -// break; -// } -// optimizedSize--; -// } output.writeVarInt(optimizedSize + 1, true); output.writeInts(object, 0, optimizedSize, false); }else{