Skip to content

Commit

Permalink
Merge pull request #247 from usethesource/test-binary-io-compression-…
Browse files Browse the repository at this point in the history
…modes

Test binary io compression modes
  • Loading branch information
jurgenvinju committed Feb 29, 2024
2 parents 4ddaf6e + fbf0f20 commit b45cd79
Show file tree
Hide file tree
Showing 6 changed files with 190 additions and 13 deletions.
5 changes: 0 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@
</includes>
</resource>
</resources>
<testResources>
<testResource>
<directory>test/main/resources</directory>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,20 @@ public void flush() throws IOException {
if (closed) {
throw new IOException("Stream closed");
}

target.flip();
if (target.hasRemaining()) {
target = flush(target);
if (!target.hasRemaining()) {
throw new IOException("flush implementation didn't correctly provide a new buffer to write to: " + target);
}
}
else {
// the buffer was empty, so flush was called on an already flushed stream
// so we'll reset the buffer to make sure we don't continue with empty buffer
target.clear();
}
assert target.hasRemaining(): "after a flush, we should have a buffer with some room. (it was: " + target + ")";
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public BinaryWireOutputStream(OutputStream stream, int stringSharingWindowSize)
this(stream, stringSharingWindowSize, 8*1024);
}
public BinaryWireOutputStream(OutputStream stream, int stringSharingWindowSize, int bufferSize) throws IOException {
assert stringSharingWindowSize > 0;
if (stream instanceof BufferedOutputStream || stream instanceof ByteBufferOutputStream) {
__stream = stream;
}
Expand Down
82 changes: 75 additions & 7 deletions src/test/java/io/usethesource/vallang/basic/BinaryIoSmokeTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,24 @@
*******************************************************************************/
package io.usethesource.vallang.basic;

import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;

import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ArgumentsSource;

import io.usethesource.vallang.ArgumentsMaxDepth;
import io.usethesource.vallang.ArgumentsMaxWidth;
import io.usethesource.vallang.ExpectedType;
Expand All @@ -42,6 +43,7 @@
import io.usethesource.vallang.io.binary.message.IValueWriter;
import io.usethesource.vallang.io.binary.stream.IValueInputStream;
import io.usethesource.vallang.io.binary.stream.IValueOutputStream;
import io.usethesource.vallang.io.binary.stream.IValueOutputStream.CompressionRate;
import io.usethesource.vallang.io.binary.util.WindowSizes;
import io.usethesource.vallang.io.binary.wire.IWireInputStream;
import io.usethesource.vallang.io.binary.wire.IWireOutputStream;
Expand All @@ -54,11 +56,44 @@

public final class BinaryIoSmokeTest extends BooleanStoreProvider {

@ParameterizedTest @ArgumentsSource(ValueProvider.class)
public void testSingleValueIO(IValueFactory vf, TypeStore ts) throws IOException {
ioRoundTrip(vf, ts, vf.integer(1));
}

@ParameterizedTest @ArgumentsSource(ValueProvider.class)
public void testSingleValue2IO(IValueFactory vf, TypeStore ts) throws IOException {
ioRoundTrip(vf, ts, vf.string("a"));
}

@ParameterizedTest @ArgumentsSource(ValueProvider.class)
public void testSmallBinaryIO(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
ioRoundTrip(vf, ts, value);
}

@ParameterizedTest @ArgumentsSource(ValueProvider.class)
void testReadReferenceSerializedFile(IValueFactory vf, TypeStore ts) throws IOException {
try (var files = new ZipInputStream(this.getClass().getResourceAsStream("/io/reference-serialized-binary-values.zip"))) {
ZipEntry current;
while ((current = files.getNextEntry()) != null) {
try (var read = new IValueInputStream(new FilterInputStream(files) {
public void close() throws IOException {
// we have to redirect the close to the entry close, not the global close
files.closeEntry();
};
}, vf, () -> ts)) {
assertNotNull(read.read());
}
catch (Throwable e) {
fail("Failed for " + current.getName(), e);
}

}
}
}



@ParameterizedTest @ArgumentsSource(ValueProvider.class)
public void testRegression40(IValueFactory vf, TypeStore store,
@GivenValue("twotups(<\\true(),twotups(<not(\\true()),and(\\false(),\\true())>,<twotups(<couples([]),\\true()>,<or([]),friends([])>),twotups(<or([]),or([])>,<or([]),\\true()>)>)>,<twotups(<not(\\true()),and(\\true(),\\true())>,<twotups(<couples([]),couples([])>,<\\true(),couples([])>),not(\\true())>),and(or([\\true()]),twotups(<or([]),\\true()>,<or([]),\\false()>))>)")
Expand Down Expand Up @@ -152,7 +187,7 @@ public void testDeepRandomValuesIO(IValueFactory vf, TypeStore ts, IValue val) t
ValueStreams.bottomupbf(val).forEach(v -> {
try {
ioRoundTrip(vf, ts, v);
} catch (IOException error) {
} catch (Throwable error) {
fail(error);
}
});
Expand Down Expand Up @@ -203,9 +238,22 @@ public void iopRoundTrip(IValueFactory vf, TypeStore ts, Type tp) throws IOExcep
}
}

private final CompressionRate[] RATES_TO_TESTS = {CompressionRate.Normal, CompressionRate.Extreme, CompressionRate.None, CompressionRate.NoSharing};

private void ioRoundTrip(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
for (var rate: RATES_TO_TESTS) {
try {
ioRoundTrip(vf, ts, value, rate);
}
catch (Throwable e) {
fail("Error with "+ rate + " compression", e);
}
}
}

private void ioRoundTrip(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
try (IValueOutputStream w = new IValueOutputStream(buffer, vf, IValueOutputStream.CompressionRate.Normal)) {
try (IValueOutputStream w = new IValueOutputStream(buffer, vf, compression)) {
w.write(value);
}
try (IValueInputStream read = new IValueInputStream(new ByteArrayInputStream(buffer.toByteArray()), vf, () -> ts)) {
Expand Down Expand Up @@ -239,10 +287,20 @@ else if (value instanceof IConstructor) {
}

private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value) throws IOException {
for (var rate: RATES_TO_TESTS) {
try {
ioRoundTripFile(vf, ts, value, rate);
}
catch (Throwable e) {
fail("Error with "+ rate + " compression", e);
}
}
}
private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws IOException {
long fileSize = 0;
File target = File.createTempFile("valllang-test-file", "something");
target.deleteOnExit();
try (IValueOutputStream w = new IValueOutputStream(FileChannel.open(target.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE), vf, IValueOutputStream.CompressionRate.Normal)) {
try (IValueOutputStream w = new IValueOutputStream(FileChannel.open(target.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE), vf, compression)) {
w.write(value);
}
fileSize = Files.size(target.toPath());
Expand All @@ -261,10 +319,20 @@ private void ioRoundTripFile(IValueFactory vf, TypeStore ts, IValue value) throw
}

private void ioRoundTripFile2(IValueFactory vf, TypeStore ts, IValue value) throws FileNotFoundException, IOException {
for (var rate: RATES_TO_TESTS) {
try {
ioRoundTripFile2(vf, ts, value, rate);
}
catch (Throwable e) {
fail("Error with "+ rate + " compression", e);
}
}
}
private void ioRoundTripFile2(IValueFactory vf, TypeStore ts, IValue value, IValueOutputStream.CompressionRate compression) throws FileNotFoundException, IOException {
long fileSize = 0;
File target = File.createTempFile("valllang-test-file", "something");
target.deleteOnExit();
try (IValueOutputStream w = new IValueOutputStream(new FileOutputStream(target), vf, IValueOutputStream.CompressionRate.Normal)) {
try (IValueOutputStream w = new IValueOutputStream(new FileOutputStream(target), vf, compression)) {
w.write(value);
}
fileSize = Files.size(target.toPath());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package io.usethesource.vallang.io;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Random;
import org.junit.jupiter.api.Test;
import io.usethesource.vallang.io.binary.util.FileChannelDirectInputStream;
import io.usethesource.vallang.io.binary.util.FileChannelDirectOutputStream;

class FileChannelOutputStreamTest {
private static final Path targetFile;
static {
Path file = null;
try {
file = Files.createTempFile("file-channel-test", "bin");
} catch (IOException e) {
System.err.println(e);
}
targetFile = file;
}

private FileChannel openWriteChannel() throws IOException {
return FileChannel.open(targetFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
}

private FileChannel openReadChannel() throws IOException {
return FileChannel.open(targetFile, StandardOpenOption.READ);
}

@Test
void testSimpleWrite() throws IOException {
roundTripChannel(new byte[][]{{42}});
}

@Test
void testBigWrite() throws IOException {
byte[] buffer = new byte[1024*1024];
new Random().nextBytes(buffer);
roundTripChannel(new byte[][]{buffer});
}

@Test
void testChunkedBigWrite() throws IOException {
byte[][] buffers = new byte[1024][];
var r = new Random();
for (int i = 0; i < buffers.length; i++) {
buffers[i] = new byte[i * 128];
r.nextBytes(buffers[i]);
}
roundTripChannel(buffers);
}


private void roundTripChannel(byte[][] buffers) throws IOException {
writeChannelInBulk(buffers);
verifyChannelInBulk(buffers);
writeChannelBytePerByte(buffers);
verifyChannelBytePerByte(buffers);
}

private void verifyChannelBytePerByte(byte[][] buffers) throws IOException {
try (var reader = new FileChannelDirectInputStream(openReadChannel())) {
for (byte[] expected: buffers) {
for (byte expect: expected) {
assertEquals(expect & 0xFF, reader.read());
}
}
}
}


private void verifyChannelInBulk(byte[][] buffers) throws IOException {
try (var reader = new FileChannelDirectInputStream(openReadChannel())) {
for (byte[] expected: buffers) {
byte[] actual = new byte[expected.length];
reader.read(actual);
assertArrayEquals(expected, actual);
}
}
}

private void writeChannelBytePerByte(byte[][] buffers) throws IOException {
try (var writer = new FileChannelDirectOutputStream(openWriteChannel(), 1)) {
for (byte[] buf: buffers) {
for (byte b: buf) {
writer.write(b);
}
}
}
}

private void writeChannelInBulk(byte[][] buffers) throws IOException {
try (var writer = new FileChannelDirectOutputStream(openWriteChannel(), 1)) {
for (byte[] buf: buffers) {
writer.write(buf);
}
}
}

}
Binary file not shown.

0 comments on commit b45cd79

Please sign in to comment.