diff --git a/src/main/java/org/scijava/io/handle/BZip2Handle.java b/src/main/java/org/scijava/io/handle/BZip2Handle.java new file mode 100644 index 000000000..c5d835552 --- /dev/null +++ b/src/main/java/org/scijava/io/handle/BZip2Handle.java @@ -0,0 +1,64 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.IOException; + +import org.scijava.io.handle.bzip2.CBZip2InputStream; +import org.scijava.io.location.AbstractCompressedHandle; +import org.scijava.io.location.BZip2Location; +import org.scijava.plugin.Plugin; + +/** + * {@link DataHandle} for a {@link BZip2Location}. + * + * @author Gabriel Einsdorf + */ +@Plugin(type = DataHandle.class) +public class BZip2Handle extends AbstractCompressedHandle { + + @Override + protected void initInputStream() throws IOException { + + int skipped = 0; + // ensure 2 bytes are skipped + while (skipped < 2) { + skipped += raw().skip(2l - skipped); + } + inputStream = new CBZip2InputStream(raw(), log()); + } + + @Override + public Class getType() { + return BZip2Location.class; + } +} diff --git a/src/main/java/org/scijava/io/handle/BufferedStreamHandle.java b/src/main/java/org/scijava/io/handle/BufferedStreamHandle.java new file mode 100644 index 000000000..717d32fcb --- /dev/null +++ b/src/main/java/org/scijava/io/handle/BufferedStreamHandle.java @@ -0,0 +1,45 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import org.scijava.io.location.Location; + +/** + * A buffered {@link StreamHandle}. + * + * @author Gabriel Einsdorf + */ +public interface BufferedStreamHandle extends + SeekableStreamHandle +{ + +} diff --git a/src/main/java/org/scijava/io/handle/DefaultBufferedStreamHandle.java b/src/main/java/org/scijava/io/handle/DefaultBufferedStreamHandle.java new file mode 100644 index 000000000..78bcecc4d --- /dev/null +++ b/src/main/java/org/scijava/io/handle/DefaultBufferedStreamHandle.java @@ -0,0 +1,239 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.scijava.io.ByteArrayByteBank; +import org.scijava.io.ByteBank; +import org.scijava.io.location.Location; + +/** + * A {@link BufferedStreamHandle} backed by a {@link ByteBank}. + * + * @author Gabriel Einsdorf + */ +public class DefaultBufferedStreamHandle extends + AbstractStreamHandle implements BufferedStreamHandle +{ + + private static final int CHUNK_SIZE = 8192; + + private final StreamHandle handle; + private ByteBank buffer; + + private InputStream inStreamProxy; + private OutputStream outStreamProxy; + + private boolean closed; + + /** + * Wraps around StreamHandle in a buffer + * + * @param handle + */ + public DefaultBufferedStreamHandle(final StreamHandle handle) { + this.handle = handle; + } + + @Override + public byte readByte() throws IOException { + + // reached end of the buffer + if (offset() == handle.length()) { + return -1; + } + + // check if we need to refill the buffer + if (offset() > maxBuf() || maxBuf() == -1) { + // buffer more bytes + final int filled = fill(); + if (filled <= 0) { + // no more bytes in input handle + return -1; + } + } + + byte b = getBufferIfOpen().getByte(offset()); + advance(1); + return b; + } + + @Override + public void seek(final long pos) throws IOException { + final long off = offset(); + if (off == pos) return; + if (pos > off) { + // ensure target is buffered + while (pos > maxBuf()) { + fill(); + } + } + // values in the range (pos < off) are already buffered + setOffset(pos); + } + + private long maxBuf() throws IOException { + return getBufferIfOpen().size() - 1; + } + + @Override + public long skip(final long n) throws IOException { + seek(offset() + n); + return handle().skip(n); + } + + @Override + public int read(final byte[] b, final int off, final int len) + throws IOException + { + + while (maxBuf() < offset() + len) { + final int filled = fill(); + if (filled <= 0) { + // no more bytes available + break; + } + } + + // read all available bytes + int available = (int) available(len); + getBufferIfOpen().getBytes(offset(), b, off, available); + setOffset(offset() + available); + return available; + } + + /** + * Fills the buffer with XX more bytes + * + * @throws IOException + */ + private int fill() throws IOException { + final byte[] buf = new byte[CHUNK_SIZE]; + final int read = handle().read(buf); + if (read <= 0) { + return -1; + } + getBufferIfOpen().appendBytes(buf, read); + return read; + } + + @Override + public InputStream in() { + if (inStreamProxy == null) { + inStreamProxy = new DataHandleInputStream<>(this); + } + return inStreamProxy; + } + + @Override + public OutputStream out() { + if (outStreamProxy == null) { + outStreamProxy = new DataHandleOutputStream<>(this); + } + return outStreamProxy; + } + + @Override + public long length() throws IOException { + return handle().length(); + } + + private StreamHandle handle() throws IOException { + if (closed) { + throw new IOException("Handle is closed!"); + } + return handle; + } + + @Override + public void setLength(final long length) throws IOException { + handle().setLength(length); + } + + @Override + public boolean isReadable() { + return !closed && handle.isReadable(); + } + + @Override + public boolean isWritable() { + return !closed && handle.isWritable(); + } + + @Override + public boolean exists() throws IOException { + return handle.exists(); + } + + @Override + public Class getType() { + return handle.getType(); + } + + @Override + public void resetStream() throws IOException { + getBufferIfOpen(); + if (handle() instanceof ResettableStreamHandle) { + ((ResettableStreamHandle) handle()).resetStream(); + } + else { + throw new IOException("Handle can not be reset!"); + } + } + + @Override + public void close() throws IOException { + if (!closed) { + closed = true; + handle().close(); + getBufferIfOpen().clear(); + buffer = null; + } + } + + /** + * @return the buffer used in this handle + * @throws IOException if this handle has been closed + */ + private ByteBank getBufferIfOpen() throws IOException { + if (closed) { + throw new IOException("Handle is closed"); + } + if (buffer == null) { + buffer = new ByteArrayByteBank(); + } + return buffer; + } +} diff --git a/src/main/java/org/scijava/io/handle/GZipHandle.java b/src/main/java/org/scijava/io/handle/GZipHandle.java new file mode 100644 index 000000000..9dac5868c --- /dev/null +++ b/src/main/java/org/scijava/io/handle/GZipHandle.java @@ -0,0 +1,68 @@ +/* + * #%L + * SCIFIO library for reading and converting scientific file formats. + * %% + * Copyright (C) 2011 - 2016 Board of Regents of the University of + * Wisconsin-Madison + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.IOException; +import java.util.zip.GZIPInputStream; + +import org.scijava.io.location.AbstractCompressedHandle; +import org.scijava.io.location.GZipLocation; +import org.scijava.plugin.Plugin; + +/** + * StreamHandle implementation for reading from gzip-compressed files or byte + * arrays. Instances of GZipHandle are read-only. + * + * @author Melissa Linkert + * @author Gabriel Einsdorf + */ +@Plugin(type = DataHandle.class) +public class GZipHandle extends AbstractCompressedHandle { + + @Override + public Class getType() { + return GZipLocation.class; + } + + @Override + protected void initInputStream() throws IOException { + inputStream = new GZIPInputStream(new DataHandleInputStream<>(raw())); + } + +// @Override +// public boolean isConstructable(final String file) throws IOException { +// final byte[] b = new byte[2]; +// s.read(b); +// s.close(); +// return Bytes.toInt(b, true) == GZIPInputStream.GZIP_MAGIC; +// } + +} diff --git a/src/main/java/org/scijava/io/handle/ZipHandle.java b/src/main/java/org/scijava/io/handle/ZipHandle.java new file mode 100644 index 000000000..a93747e94 --- /dev/null +++ b/src/main/java/org/scijava/io/handle/ZipHandle.java @@ -0,0 +1,201 @@ +/* + * #%L + * SCIFIO library for reading and converting scientific file formats. + * %% + * Copyright (C) 2011 - 2016 Board of Regents of the University of + * Wisconsin-Madison + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.File; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.scijava.io.location.AbstractCompressedHandle; +import org.scijava.io.location.Location; +import org.scijava.io.location.ZipLocation; +import org.scijava.plugin.Plugin; + +/** + * StreamHandle implementation for reading from Zip-compressed files or byte + * arrays. Instances of ZipHandle are read-only. + * + * @see StreamHandle + * @author Melissa Linkert + * @author Gabriel Einsdorf + */ +@Plugin(type = DataHandle.class) +public class ZipHandle extends AbstractCompressedHandle { + + // -- Fields -- + + private DataHandle in; + + private String entryName; + + private ZipEntry entry; + + private long entryLength = -1l; + + // -- ZipHandle API methods -- + + /** Get the name of the backing Zip entry. */ + public String getEntryName() { + return entryName; + } + + @Override + public void resetStream() throws IOException { + + if (raw() instanceof ResettableStreamHandle) { + ((ResettableStreamHandle) raw()).resetStream(); + } + else { + raw().seek(0l); + } + + inputStream = new ZipInputStream(new DataHandleInputStream<>(raw())); + // FIXME add Buffering + + seekToEntry(); + + } + + // -- IRandomAccess API methods -- + + @Override + public void close() throws IOException { + inputStream = null; + entryName = null; + entryLength = -1; + if (in != null) in.close(); + in = null; + } + + // -- Helper methods -- + + /** + * Seeks to the relevant ZIP entry, populating the stream length accordingly. + */ + private void seekToEntry() throws IOException { + + while (true) { + final ZipEntry e = ((ZipInputStream) inputStream).getNextEntry(); + if (entryName == null) { + entry = e; + entryName = e.getName(); + } + if (entryName.equals(e.getName())) { + // found the matching entry name (or first entry if the name is + // null) + if (entryLength < 0) { + final boolean resetNeeded = populateLength(e.getSize()); + if (resetNeeded) { + // stream length was calculated by force, need to reset + resetStream(); + } + } + break; + } + } + } + + /** + * Sets the stream length, computing it by force if necessary. + * + * @return if the Stream needs to be reset + */ + private boolean populateLength(final long size) throws IOException { + if (size >= 0) { + entryLength = size; + return false; + } + // size is unknown, so we must read the stream manually + long length = 0; + final DataHandle stream = raw(); + while (true) { + final long skipped = stream.skip(Long.MAX_VALUE); + if (skipped == 0) { + // NB: End of stream, we hope. Technically there is no contract + // for when skip(long) returns 0, but in practice it seems to be + // when end of stream is reached. + break; + } + length += skipped; + } + + entryLength = length; + return true; + } + + @Override + public Class getType() { + return ZipLocation.class; + } + + @Override + protected void initInputStream() throws IOException { + inputStream = new ZipInputStream(new DataHandleInputStream<>(raw())); + + entry = get().getEntry(); + if (entry == null) { + // strip off .zip extension and directory prefix + final String n = raw().get().getName(); + String name = n.substring(0, n.length() - 4); + + int slash = name.lastIndexOf(File.separator); + if (slash < 0) slash = name.lastIndexOf("/"); + if (slash >= 0) name = name.substring(slash + 1); + + // look for Zip entry with same prefix as the Zip file itself + boolean matchFound = false; + ZipEntry ze; + while ((ze = ((ZipInputStream) inputStream).getNextEntry()) != null) { + if (entryName == null) entryName = ze.getName(); + if (!matchFound && ze.getName().startsWith(name)) { + // found entry with matching name + entryName = ze.getName(); + entry = ze; + matchFound = true; + } + } + resetStream(); + } + } + + @Override + public long length() throws IOException { + if (entry == null) { + return -1; + } + return entry.getSize(); + } + + public long getEntryLength() { + return entryLength; + } +} diff --git a/src/main/java/org/scijava/io/handle/bzip2/CBZip2InputStream.java b/src/main/java/org/scijava/io/handle/bzip2/CBZip2InputStream.java new file mode 100644 index 000000000..fd7cbcc9a --- /dev/null +++ b/src/main/java/org/scijava/io/handle/bzip2/CBZip2InputStream.java @@ -0,0 +1,1028 @@ +/* + * #%L + * SCIFIO library for reading and converting scientific file formats. + * %% + * Copyright (C) 2011 - 2016 Board of Regents of the University of + * Wisconsin-Madison + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * to whom the Ant project is very grateful for his + * great code. + */ + +package org.scijava.io.handle.bzip2; + +import java.io.IOException; +import java.io.InputStream; + +import org.scijava.io.handle.DataHandle; +import org.scijava.io.location.Location; +import org.scijava.log.LogService; + +/** + * An input stream that decompresses from the BZip2 format (without the file + * header chars) to be read as any other stream. + *

+ * The decompression requires large amounts of memory. Thus you should call the + * {@link #close()} method as soon as possible, to force + * CBZip2InputStream to release the allocated memory. + *

+ *

+ * CBZip2InputStream reads bytes from the compressed source stream via + * the single byte {@link java.io.InputStream#read() read()} method exclusively. + * Thus you should consider to use a buffered source stream. + *

+ *

+ * Instances of this class are not threadsafe. + *

+ */ +public class CBZip2InputStream extends InputStream { + + // -- Constants -- + + private static final int BASE_BLOCK_SIZE = 100000; + + private static final int MAX_ALPHA_SIZE = 258; + + private static final int MAX_CODE_LEN = 23; + + private static final int RUNA = 0; + + private static final int RUNB = 1; + + private static final int N_GROUPS = 6; + + private static final int G_SIZE = 50; + + private static final int MAX_SELECTORS = (2 + (900000 / G_SIZE)); + + private static final int[] R_NUMS = { 619, 720, 127, 481, 931, 816, 813, 233, + 566, 247, 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, 733, 859, 335, + 708, 621, 574, 73, 654, 730, 472, 419, 436, 278, 496, 867, 210, 399, 680, + 480, 51, 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, 862, 687, 507, + 283, 482, 129, 807, 591, 733, 623, 150, 238, 59, 379, 684, 877, 625, 169, + 643, 105, 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, 73, 122, 335, + 530, 442, 853, 695, 249, 445, 515, 909, 545, 703, 919, 874, 474, 882, 500, + 594, 612, 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, 161, 604, 958, + 533, 221, 400, 386, 867, 600, 782, 382, 596, 414, 171, 516, 375, 682, 485, + 911, 276, 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, 227, 730, 475, + 186, 263, 647, 537, 686, 600, 224, 469, 68, 770, 919, 190, 373, 294, 822, + 808, 206, 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, 715, 67, 618, + 276, 204, 918, 873, 777, 604, 560, 951, 160, 578, 722, 79, 804, 96, 409, + 713, 940, 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, 645, 863, 803, + 350, 139, 93, 354, 99, 820, 908, 609, 772, 154, 274, 580, 184, 79, 626, 630, + 742, 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, 411, 521, 938, 300, + 821, 78, 343, 175, 128, 250, 170, 774, 972, 275, 999, 639, 495, 78, 352, + 126, 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, 669, 112, 134, 694, + 363, 992, 809, 743, 168, 974, 944, 375, 748, 52, 600, 747, 642, 182, 862, + 81, 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, 897, 955, 664, 981, + 649, 113, 974, 459, 893, 228, 433, 837, 553, 268, 926, 240, 102, 654, 459, + 51, 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, 946, 670, 656, 610, + 738, 392, 760, 799, 887, 653, 978, 321, 576, 617, 626, 502, 894, 679, 243, + 440, 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, 707, 151, 457, 449, + 797, 195, 791, 558, 945, 679, 297, 59, 87, 824, 713, 663, 412, 693, 342, + 606, 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, 343, 97, 430, 751, + 497, 314, 983, 374, 822, 928, 140, 206, 73, 263, 980, 736, 876, 478, 430, + 305, 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, 369, 970, 294, 750, + 807, 827, 150, 790, 288, 923, 804, 378, 215, 828, 592, 281, 565, 555, 710, + 82, 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, 661, 821, 976, 991, + 658, 869, 905, 758, 745, 193, 768, 550, 608, 933, 378, 286, 215, 979, 792, + 961, 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, 372, 567, 466, 434, + 645, 210, 389, 550, 919, 135, 780, 773, 635, 389, 707, 100, 626, 958, 165, + 504, 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, 645, 990, 626, 197, + 510, 357, 358, 850, 858, 364, 936, 638 }; + + // -- Helper methods -- + + private void reportCRCError() { + // The clean way would be to throw an exception. + // throw new IOException("crc error"); + + // Just print a message, like the previous versions of this class did + log.error("BZip2 CRC error"); + } + + private void makeMaps() { + final boolean[] inUse = this.data.inUse; + final byte[] seqToUnseq = this.data.seqToUnseq; + + int nInUseShadow = 0; + + for (int i = 0; i < 256; i++) { + if (inUse[i]) seqToUnseq[nInUseShadow++] = (byte) i; + } + + this.nInUse = nInUseShadow; + } + + // -- Fields -- + + /** + * Index of the last char in the block, so the block size == last + 1. + */ + private int last; + + /** + * Index in zptr[] of original string after sorting. + */ + private int origPtr; + + /** + * always: in the range 0 .. 9. The current block size is 100000 * this + * number. + */ + private int blockSize100k; + + private boolean blockRandomised; + + private int bsBuff; + + private int bsLive; + + private final CRC crc = new CRC(); + + private int nInUse; + + private DataHandle in; + + private final LogService log; + + private int currentChar = -1; + + private static final int EOF = 0; + + private static final int START_BLOCK_STATE = 1; + + private static final int RAND_PART_A_STATE = 2; + + private static final int RAND_PART_B_STATE = 3; + + private static final int RAND_PART_C_STATE = 4; + + private static final int NO_RAND_PART_A_STATE = 5; + + private static final int NO_RAND_PART_B_STATE = 6; + + private static final int NO_RAND_PART_C_STATE = 7; + + private int currentState = START_BLOCK_STATE; + + private int storedBlockCRC, storedCombinedCRC; + + private int computedBlockCRC, computedCombinedCRC; + + // Variables used by setup* methods exclusively + + private int suCount; + + private int suCh2; + + private int suChPrev; + + private int suI2; + + private int suJ2; + + private int suRNToGo; + + private int suRTPos; + + private int suTPos; + + private char suZ; + + /** + * All memory intensive stuff. This field is initialized by initBlock(). + */ + private CBZip2InputStream.Data data; + + /** + * Constructs a new CBZip2InputStream which decompresses bytes read from the + * specified stream. + *

+ * Although BZip2 headers are marked with the magic "Bz" this + * constructor expects the next byte in the stream to be the first one after + * the magic. Thus callers have to skip the first two bytes. Otherwise this + * constructor will throw an exception. + *

+ * + * @throws IOException if the stream content is malformed or an I/O error + * occurs. + * @throws NullPointerException if in == null + */ + public CBZip2InputStream(final DataHandle in, final LogService log) + throws IOException + { + super(); + + this.in = in; + this.log = log; + init(); + } + + @Override + public int read() throws IOException { + if (this.in != null) return read0(); + throw new IOException("stream closed"); + } + + @Override + public int read(final byte[] dest, final int offs, final int len) + throws IOException + { + if (offs < 0) { + throw new IndexOutOfBoundsException("offs(" + offs + ") < 0."); + } + if (len < 0) { + throw new IndexOutOfBoundsException("len(" + len + ") < 0."); + } + if (offs + len > dest.length) { + throw new IndexOutOfBoundsException("offs(" + offs + ") + len(" + len + + ") > dest.length(" + dest.length + ")."); + } + if (this.in == null) throw new IOException("stream closed"); + + final int hi = offs + len; + int destOffs = offs; + for (int b; (destOffs < hi) && ((b = read0()) >= 0);) { + dest[destOffs++] = (byte) b; + } + + return (destOffs == offs) ? -1 : (destOffs - offs); + } + + private int read0() throws IOException { + final int retChar = this.currentChar; + + switch (this.currentState) { + case EOF: + return -1; + + case START_BLOCK_STATE: + throw new IllegalStateException(); + + case RAND_PART_A_STATE: + throw new IllegalStateException(); + + case RAND_PART_B_STATE: + setupRandPartB(); + break; + + case RAND_PART_C_STATE: + setupRandPartC(); + break; + + case NO_RAND_PART_A_STATE: + throw new IllegalStateException(); + + case NO_RAND_PART_B_STATE: + setupNoRandPartB(); + break; + + case NO_RAND_PART_C_STATE: + setupNoRandPartC(); + break; + + default: + throw new IllegalStateException(); + } + + return retChar; + } + + private void init() throws IOException { + final int magic2 = this.in.read(); + if (magic2 != 'h') { + throw new IOException("Stream is not BZip2 formatted: expected 'h'" + + " as first byte but got '" + (char) magic2 + "'"); + } + + final int blockSize = this.in.read(); + if ((blockSize < '1') || (blockSize > '9')) { + throw new IOException("Stream is not BZip2 formatted: illegal " + + "blocksize " + (char) blockSize); + } + + this.blockSize100k = blockSize - '0'; + + initBlock(); + setupBlock(); + } + + private void initBlock() throws IOException { + final char magic0 = bsGetUByte(); + final char magic1 = bsGetUByte(); + final char magic2 = bsGetUByte(); + final char magic3 = bsGetUByte(); + final char magic4 = bsGetUByte(); + final char magic5 = bsGetUByte(); + + if (magic0 == 0x17 && magic1 == 0x72 && magic2 == 0x45 && magic3 == 0x38 && + magic4 == 0x50 && magic5 == 0x90) + { + complete(); // end of file + } + else if (magic0 != 0x31 || // '1' + magic1 != 0x41 || // ')' + magic2 != 0x59 || // 'Y' + magic3 != 0x26 || // '&' + magic4 != 0x53 || // 'S' + magic5 != 0x59) // 'Y' + { + this.currentState = EOF; + throw new IOException("bad block header"); + } + else { + this.storedBlockCRC = bsGetInt(); + this.blockRandomised = bsR(1) == 1; + + // Allocate data here instead in constructor, so we do not + // allocate it if the input file is empty. + if (this.data == null) { + this.data = new Data(this.blockSize100k); + } + + // currBlockNo++; + getAndMoveToFrontDecode(); + + this.crc.initialiseCRC(); + this.currentState = START_BLOCK_STATE; + } + } + + private void endBlock() { + this.computedBlockCRC = this.crc.getFinalCRC(); + + // A bad CRC is considered a fatal error. + if (this.storedBlockCRC != this.computedBlockCRC) { + // make next blocks readable without error + // (repair feature, not yet documented, not tested) + this.computedCombinedCRC = (this.storedCombinedCRC << 1) | + (this.storedCombinedCRC >>> 31); + this.computedCombinedCRC ^= this.storedBlockCRC; + + reportCRCError(); + } + + this.computedCombinedCRC = (this.computedCombinedCRC << 1) | + (this.computedCombinedCRC >>> 31); + this.computedCombinedCRC ^= this.computedBlockCRC; + } + + private void complete() throws IOException { + this.storedCombinedCRC = bsGetInt(); + this.currentState = EOF; + this.data = null; + + if (this.storedCombinedCRC != this.computedCombinedCRC) { + reportCRCError(); + } + } + + @Override + public void close() throws IOException { + if (in != null) { + try { + if (in != System.in) in.close(); + } + finally { + this.data = null; + this.in = null; + } + } + } + + private int bsR(final int n) throws IOException { + int bsLiveShadow = this.bsLive; + int bsBuffShadow = this.bsBuff; + + if (bsLiveShadow < n) { + do { + final int thech = in.read(); + + if (thech < 0) throw new IOException("unexpected end of stream"); + + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + } + while (bsLiveShadow < n); + + this.bsBuff = bsBuffShadow; + } + + this.bsLive = bsLiveShadow - n; + return (bsBuffShadow >> (bsLiveShadow - n)) & ((1 << n) - 1); + } + + private boolean bsGetBit() throws IOException { + return bsR(1) != 0; + } + + private char bsGetUByte() throws IOException { + return (char) bsR(8); + } + + private int bsGetInt() throws IOException { + return (((((bsR(8) << 8) | bsR(8)) << 8) | bsR(8)) << 8) | bsR(8); + } + + /** Called by createHuffmanDecodingTables() exclusively. */ + private static void hbCreateDecodeTables(final int[] limit, final int[] base, + final int[] perm, final char[] length, final int minLen, final int maxLen, + final int alphaSize) + { + for (int i = minLen, pp = 0; i <= maxLen; i++) { + for (int j = 0; j < alphaSize; j++) { + if (length[j] == i) perm[pp++] = j; + } + } + + for (int i = MAX_CODE_LEN; --i > 0;) { + base[i] = 0; + limit[i] = 0; + } + + for (int i = 0; i < alphaSize; i++) { + base[length[i] + 1]++; + } + + for (int i = 1, b = base[0]; i < MAX_CODE_LEN; i++) { + b += base[i]; + base[i] = b; + } + + for (int i = minLen, vec = 0, b = base[i]; i <= maxLen; i++) { + final int nb = base[i + 1]; + vec += nb - b; + b = nb; + limit[i] = vec - 1; + vec <<= 1; + } + + for (int i = minLen + 1; i <= maxLen; i++) { + base[i] = ((limit[i - 1] + 1) << 1) - base[i]; + } + } + + private void recvDecodingTables() throws IOException { + final Data dataShadow = this.data; + final boolean[] inUse = dataShadow.inUse; + final byte[] pos = dataShadow.recvDecodingTablesPos; + final byte[] selector = dataShadow.selector; + final byte[] selectorMtf = dataShadow.selectorMtf; + + int inUse16 = 0; + + // Receive the mapping table + for (int i = 0; i < 16; i++) { + if (bsGetBit()) { + inUse16 |= 1 << i; + } + } + + for (int i = 256; --i >= 0;) + inUse[i] = false; + + for (int i = 0; i < 16; i++) { + if ((inUse16 & (1 << i)) != 0) { + final int i16 = i << 4; + for (int j = 0; j < 16; j++) { + if (bsGetBit()) inUse[i16 + j] = true; + } + } + } + + makeMaps(); + final int alphaSize = this.nInUse + 2; + + // Now the selectors + final int nGroups = bsR(3); + final int nSelectors = bsR(15); + + for (int i = 0; i < nSelectors; i++) { + int j = 0; + while (bsGetBit()) + j++; + selectorMtf[i] = (byte) j; + } + + // Undo the MTF values for the selectors. + for (int v = nGroups; --v >= 0;) + pos[v] = (byte) v; + + for (int i = 0; i < nSelectors; i++) { + int v = selectorMtf[i] & 0xff; + final byte tmp = pos[v]; + while (v > 0) { + // nearly all times v is zero, 4 in most other cases + pos[v] = pos[v - 1]; + v--; + } + pos[0] = tmp; + selector[i] = tmp; + } + + final char[][] len = dataShadow.tempCharArray2d; + + // Now the coding tables + for (int t = 0; t < nGroups; t++) { + int curr = bsR(5); + final char[] tLen = len[t]; + for (int i = 0; i < alphaSize; i++) { + while (bsGetBit()) + curr += bsGetBit() ? -1 : 1; + tLen[i] = (char) curr; + } + } + + // finally create the Huffman tables + createHuffmanDecodingTables(alphaSize, nGroups); + } + + /** Called by recvDecodingTables() exclusively. */ + private void createHuffmanDecodingTables(final int alphaSize, + final int nGroups) + { + final Data dataShadow = this.data; + final char[][] len = dataShadow.tempCharArray2d; + final int[] minLens = dataShadow.minLens; + final int[][] limit = dataShadow.limit; + final int[][] base = dataShadow.base; + final int[][] perm = dataShadow.perm; + + for (int t = 0; t < nGroups; t++) { + int minLen = 32; + int maxLen = 0; + final char[] tLen = len[t]; + for (int i = alphaSize; --i >= 0;) { + final char lent = tLen[i]; + if (lent > maxLen) maxLen = lent; + if (lent < minLen) minLen = lent; + } + hbCreateDecodeTables(limit[t], base[t], perm[t], len[t], minLen, maxLen, + alphaSize); + minLens[t] = minLen; + } + } + + private void getAndMoveToFrontDecode() throws IOException { + this.origPtr = bsR(24); + recvDecodingTables(); + + final Data dataShadow = this.data; + final byte[] ll8 = dataShadow.ll8; + final int[] unzftab = dataShadow.unzftab; + final byte[] selector = dataShadow.selector; + final byte[] seqToUnseq = dataShadow.seqToUnseq; + final char[] yy = dataShadow.getAndMoveToFrontDecodeYY; + final int[] minLens = dataShadow.minLens; + final int[][] limit = dataShadow.limit; + final int[][] base = dataShadow.base; + final int[][] perm = dataShadow.perm; + final int limitLast = this.blockSize100k * 100000; + + // Setting up the unzftab entries here is not strictly + // necessary, but it does save having to do it later + // in a separate pass, and so saves a block's worth of + // cache misses. + for (int i = 256; --i >= 0;) { + yy[i] = (char) i; + unzftab[i] = 0; + } + + int groupNo = 0; + int groupPos = G_SIZE - 1; + final int eob = this.nInUse + 1; + int nextSym = getAndMoveToFrontDecode0(0); + int bsBuffShadow = this.bsBuff; + int bsLiveShadow = this.bsLive; + int lastShadow = -1; + int zt = selector[groupNo] & 0xff; + int[] baseZT = base[zt]; + int[] limitZT = limit[zt]; + int[] permZT = perm[zt]; + int minLensZT = minLens[zt]; + + while (nextSym != eob) { + if ((nextSym == RUNA) || (nextSym == RUNB)) { + int s = -1; + + for (int n = 1; true; n <<= 1) { + if (nextSym == RUNA) s += n; + else if (nextSym == RUNB) s += n << 1; + else break; + + if (groupPos == 0) { + groupPos = G_SIZE - 1; + zt = selector[++groupNo] & 0xff; + baseZT = base[zt]; + limitZT = limit[zt]; + permZT = perm[zt]; + minLensZT = minLens[zt]; + } + else groupPos--; + + int zn = minLensZT; + + // Inlined: + // int zvec = bsR(zn); + while (bsLiveShadow < zn) { + final int thech = in.read(); + if (thech >= 0) { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + throw new IOException("unexpected end of stream"); + } + int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) & ((1 << zn) - 1); + bsLiveShadow -= zn; + + while (zvec > limitZT[zn]) { + zn++; + while (bsLiveShadow < 1) { + final int thech = in.read(); + if (thech >= 0) { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + throw new IOException("unexpected end of stream"); + } + bsLiveShadow--; + zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); + } + nextSym = permZT[zvec - baseZT[zn]]; + } + + final byte ch = seqToUnseq[yy[0]]; + unzftab[ch & 0xff] += s + 1; + + while (s-- >= 0) + ll8[++lastShadow] = ch; + + if (lastShadow >= limitLast) throw new IOException("block overrun"); + } + else { + if (++lastShadow >= limitLast) { + throw new IOException("block overrun"); + } + + final char tmp = yy[nextSym - 1]; + unzftab[seqToUnseq[tmp] & 0xff]++; + ll8[lastShadow] = seqToUnseq[tmp]; + + /* + * This loop is hammered during decompression, hence avoid + * native method call overhead of System.arraycopy for very + * small ranges to copy. + */ + if (nextSym <= 16) { + for (int j = nextSym - 1; j > 0;) + yy[j] = yy[--j]; + } + else System.arraycopy(yy, 0, yy, 1, nextSym - 1); + + yy[0] = tmp; + + if (groupPos == 0) { + groupPos = G_SIZE - 1; + zt = selector[++groupNo] & 0xff; + baseZT = base[zt]; + limitZT = limit[zt]; + permZT = perm[zt]; + minLensZT = minLens[zt]; + } + else groupPos--; + + int zn = minLensZT; + + // Inlined: + // int zvec = bsR(zn); + while (bsLiveShadow < zn) { + final int thech = in.read(); + if (thech >= 0) { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + throw new IOException("unexpected end of stream"); + } + int zvec = (bsBuffShadow >> (bsLiveShadow - zn)) & ((1 << zn) - 1); + bsLiveShadow -= zn; + + while (zvec > limitZT[zn]) { + zn++; + while (bsLiveShadow < 1) { + final int thech = in.read(); + if (thech >= 0) { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + throw new IOException("unexpected end of stream"); + } + bsLiveShadow--; + zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); + } + nextSym = permZT[zvec - baseZT[zn]]; + } + } + + this.last = lastShadow; + this.bsLive = bsLiveShadow; + this.bsBuff = bsBuffShadow; + } + + private int getAndMoveToFrontDecode0(final int groupNo) throws IOException { + final Data dataShadow = this.data; + final int zt = dataShadow.selector[groupNo] & 0xff; + final int[] limitZT = dataShadow.limit[zt]; + int zn = dataShadow.minLens[zt]; + int zvec = bsR(zn); + int bsLiveShadow = this.bsLive; + int bsBuffShadow = this.bsBuff; + + while (zvec > limitZT[zn]) { + zn++; + while (bsLiveShadow < 1) { + final int thech = in.read(); + + if (thech >= 0) { + bsBuffShadow = (bsBuffShadow << 8) | thech; + bsLiveShadow += 8; + continue; + } + throw new IOException("unexpected end of stream"); + } + bsLiveShadow--; + zvec = (zvec << 1) | ((bsBuffShadow >> bsLiveShadow) & 1); + } + + this.bsLive = bsLiveShadow; + this.bsBuff = bsBuffShadow; + + return dataShadow.perm[zt][zvec - dataShadow.base[zt][zn]]; + } + + private void setupBlock() throws IOException { + if (this.data == null) return; + + final int[] cftab = this.data.cftab; + final int[] tt = this.data.initTT(this.last + 1); + final byte[] ll8 = this.data.ll8; + cftab[0] = 0; + System.arraycopy(this.data.unzftab, 0, cftab, 1, 256); + + for (int i = 1, c = cftab[0]; i <= 256; i++) { + c += cftab[i]; + cftab[i] = c; + } + + for (int i = 0, lastShadow = this.last; i <= lastShadow; i++) { + tt[cftab[ll8[i] & 0xff]++] = i; + } + + if ((this.origPtr < 0) || (this.origPtr >= tt.length)) { + throw new IOException("stream corrupted"); + } + + this.suTPos = tt[this.origPtr]; + this.suCount = 0; + this.suI2 = 0; + this.suCh2 = 256; /* not a char and not EOF */ + + if (this.blockRandomised) { + this.suRNToGo = 0; + this.suRTPos = 0; + setupRandPartA(); + } + else setupNoRandPartA(); + } + + private void setupRandPartA() throws IOException { + if (this.suI2 <= this.last) { + this.suChPrev = this.suCh2; + int suCh2Shadow = this.data.ll8[this.suTPos] & 0xff; + this.suTPos = this.data.tt[this.suTPos]; + if (this.suRNToGo == 0) { + this.suRNToGo = R_NUMS[this.suRTPos] - 1; + if (++this.suRTPos == 512) this.suRTPos = 0; + } + else this.suRNToGo--; + this.suCh2 = suCh2Shadow ^= (this.suRNToGo == 1) ? 1 : 0; + this.suI2++; + this.currentChar = suCh2Shadow; + this.currentState = RAND_PART_B_STATE; + this.crc.updateCRC(suCh2Shadow); + } + else { + endBlock(); + initBlock(); + setupBlock(); + } + } + + private void setupNoRandPartA() throws IOException { + if (this.suI2 <= this.last) { + this.suChPrev = this.suCh2; + final int suCh2Shadow = this.data.ll8[this.suTPos] & 0xff; + this.suCh2 = suCh2Shadow; + this.suTPos = this.data.tt[this.suTPos]; + this.suI2++; + this.currentChar = suCh2Shadow; + this.currentState = NO_RAND_PART_B_STATE; + this.crc.updateCRC(suCh2Shadow); + } + else { + this.currentState = NO_RAND_PART_A_STATE; + endBlock(); + initBlock(); + setupBlock(); + } + } + + private void setupRandPartB() throws IOException { + if (this.suCh2 != this.suChPrev) { + this.currentState = RAND_PART_A_STATE; + this.suCount = 1; + setupRandPartA(); + } + else if (++this.suCount >= 4) { + this.suZ = (char) (this.data.ll8[this.suTPos] & 0xff); + this.suTPos = this.data.tt[this.suTPos]; + if (this.suRNToGo == 0) { + this.suRNToGo = R_NUMS[this.suRTPos] - 1; + if (++this.suRTPos == 512) { + this.suRTPos = 0; + } + } + else this.suRNToGo--; + this.suJ2 = 0; + this.currentState = RAND_PART_C_STATE; + if (this.suRNToGo == 1) this.suZ ^= 1; + setupRandPartC(); + } + else { + this.currentState = RAND_PART_A_STATE; + setupRandPartA(); + } + } + + private void setupRandPartC() throws IOException { + if (this.suJ2 < this.suZ) { + this.currentChar = this.suCh2; + this.crc.updateCRC(this.suCh2); + this.suJ2++; + } + else { + this.currentState = RAND_PART_A_STATE; + this.suI2++; + this.suCount = 0; + setupRandPartA(); + } + } + + private void setupNoRandPartB() throws IOException { + if (this.suCh2 != this.suChPrev) { + this.suCount = 1; + setupNoRandPartA(); + } + else if (++this.suCount >= 4) { + this.suZ = (char) (this.data.ll8[this.suTPos] & 0xff); + this.suTPos = this.data.tt[this.suTPos]; + this.suJ2 = 0; + setupNoRandPartC(); + } + else setupNoRandPartA(); + } + + private void setupNoRandPartC() throws IOException { + if (this.suJ2 < this.suZ) { + final int suCh2Shadow = this.suCh2; + this.currentChar = suCh2Shadow; + this.crc.updateCRC(suCh2Shadow); + this.suJ2++; + this.currentState = NO_RAND_PART_C_STATE; + } + else { + this.suI2++; + this.suCount = 0; + setupNoRandPartA(); + } + } + + private static final class Data extends Object { + + // (with blockSize 900k) + final boolean[] inUse = new boolean[256]; // 256 byte + + final byte[] seqToUnseq = new byte[256]; // 256 byte + + final byte[] selector = new byte[MAX_SELECTORS]; // 18002 byte + + final byte[] selectorMtf = new byte[MAX_SELECTORS]; // 18002 byte + + /** + * Freq table collected to save a pass over the data during decompression. + */ + final int[] unzftab = new int[256]; // 1024 byte + + final int[][] limit = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 + // byte + + final int[][] base = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte + + final int[][] perm = new int[N_GROUPS][MAX_ALPHA_SIZE]; // 6192 byte + + final int[] minLens = new int[N_GROUPS]; // 24 byte + + final int[] cftab = new int[257]; // 1028 byte + + final char[] getAndMoveToFrontDecodeYY = new char[256]; // 512 byte + + // 3096 byte + final char[][] tempCharArray2d = new char[N_GROUPS][MAX_ALPHA_SIZE]; + + final byte[] recvDecodingTablesPos = new byte[N_GROUPS]; // 6 byte + + // --------------- + // 60798 byte + + int[] tt; // 3600000 byte + + byte[] ll8; // 900000 byte + + // --------------- + // 4560782 byte + // =============== + + Data(final int blockSize100k) { + super(); + + this.ll8 = new byte[blockSize100k * BASE_BLOCK_SIZE]; + } + + /** + * Initializes the {@link #tt} array. This method is called when the + * required length of the array is known. I don't initialize it at + * construction time to avoid unneccessary memory allocation when + * compressing small files. + */ + int[] initTT(final int length) { + int[] ttShadow = this.tt; + + // tt.length should always be >= length, but theoretically + // it can happen, if the compressor mixed small and large + // blocks. Normally only the last block will be smaller + // than others. + if ((ttShadow == null) || (ttShadow.length < length)) { + this.tt = ttShadow = new int[length]; + } + + return ttShadow; + } + } + +} diff --git a/src/main/java/org/scijava/io/handle/bzip2/CRC.java b/src/main/java/org/scijava/io/handle/bzip2/CRC.java new file mode 100644 index 000000000..9f8c5bd9c --- /dev/null +++ b/src/main/java/org/scijava/io/handle/bzip2/CRC.java @@ -0,0 +1,141 @@ +/* + * #%L + * SCIFIO library for reading and converting scientific file formats. + * %% + * Copyright (C) 2011 - 2016 Board of Regents of the University of + * Wisconsin-Madison + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +/* + * Copyright 2001-2002,2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * to whom the Ant project is very grateful for his + * great code. + */ + +package org.scijava.io.handle.bzip2; + +/** Holds and calculates a CRC for verifying data integrity. */ +public class CRC { + + // -- Constants -- + + public static final int[] CRC_32_TABLE = { 0x00000000, 0x04c11db7, 0x09823b6e, + 0x0d4326d9, 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, 0x2608edb8, + 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, + 0x384fbdbd, 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, 0x5f15adac, + 0x5bd4b01b, 0x569796c2, 0x52568b75, 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, + 0x675a1011, 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, 0x9823b6e0, + 0x9ce2ab57, 0x91a18d8e, 0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, + 0x8664e6e5, 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84, + 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, 0xd4326d90, 0xd0f37027, 0xddb056fe, + 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, 0xf23a8028, + 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, + 0xec7dd02d, 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, + 0x23431b1c, 0x2e003dc5, 0x2ac12072, 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, + 0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, 0x7897ab07, + 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, + 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, 0x4d9b3063, + 0x495a2dd4, 0x44190b0d, 0x40d816ba, 0xaca5c697, 0xa864db20, 0xa527fdf9, + 0xa1e6e04e, 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, 0x8aad2b2f, + 0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044, 0x902b669d, + 0x94ea7b2a, 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, 0xf3b06b3b, + 0xf771768c, 0xfa325055, 0xfef34de2, 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, + 0xcbffd686, 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, 0x690ce0ee, + 0x6dcdfd59, 0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c, + 0x774bb0eb, 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, 0x5c007b8a, + 0x58c1663d, 0x558240e4, 0x51435d53, 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, + 0x285e1d47, 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, 0x0315d626, + 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, + 0x1d528623, 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2, + 0xe6ea3d65, 0xeba91bbc, 0xef68060b, 0xd727bbb6, 0xd3e6a601, 0xdea580d8, + 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, 0xbd3e8d7e, + 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, + 0xa379dd7b, 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, + 0x8cf30bad, 0x81b02d74, 0x857130c3, 0x5d8a9099, 0x594b8d2e, 0x5408abf7, + 0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, 0x7b827d21, + 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd, 0x6c47164a, 0x61043093, + 0x65c52d24, 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, 0x029f3d35, + 0x065e2082, 0x0b1d065b, 0x0fdc1bec, 0x3793a651, 0x3352bbe6, 0x3e119d3f, + 0x3ad08088, 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, 0xc5a92679, + 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, + 0xdbee767c, 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, 0xf0a5bd1d, + 0xf464a0aa, 0xf9278673, 0xfde69bc4, 0x89b8fd09, 0x8d79e0be, 0x803ac667, + 0x84fbdbd0, 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, 0xafb010b1, + 0xab710d06, 0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03, + 0xb1f740b4 }; + + // -- Fields -- + + private int globalCrc; + + // -- Constructor -- + + public CRC() { + initialiseCRC(); + } + + // -- CRC API methods -- + + public void initialiseCRC() { + globalCrc = 0xffffffff; + } + + public int getFinalCRC() { + return ~globalCrc; + } + + public int getGlobalCRC() { + return globalCrc; + } + + public void setGlobalCRC(final int newCrc) { + globalCrc = newCrc; + } + + public void updateCRC(final int inCh) { + int temp = (globalCrc >> 24) ^ inCh; + if (temp < 0) { + temp = 256 + temp; + } + globalCrc = (globalCrc << 8) ^ CRC.CRC_32_TABLE[temp]; + } + +} diff --git a/src/main/java/org/scijava/io/location/AbstractCompressedHandle.java b/src/main/java/org/scijava/io/location/AbstractCompressedHandle.java new file mode 100644 index 000000000..d7bc194d7 --- /dev/null +++ b/src/main/java/org/scijava/io/location/AbstractCompressedHandle.java @@ -0,0 +1,133 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.scijava.io.handle.AbstractStreamHandle; +import org.scijava.io.handle.DataHandle; +import org.scijava.io.handle.DataHandleService; +import org.scijava.io.handle.ResettableStreamHandle; +import org.scijava.plugin.Parameter; + +/** + * Abstract superclass for {@link DataHandle}s that operate on compressed data. + * + * @author Gabriel Einsdorf + */ +public abstract class AbstractCompressedHandle + extends AbstractStreamHandle implements ResettableStreamHandle +{ + + private DataHandle rawHandle; + protected InputStream inputStream; + + @Parameter + private DataHandleService dataHandleService; + + public AbstractCompressedHandle() { + super(); + } + + @Override + public void resetStream() throws IOException { + + if (raw() instanceof ResettableStreamHandle) { + ((ResettableStreamHandle) rawHandle).resetStream(); + } + else { + rawHandle.seek(0); + } + initInputStream(); + } + + @Override + public InputStream in() throws IOException { + if (inputStream == null) { + initInputStream(); + } + return inputStream; + } + + @Override + public long skip(long n) throws IOException { + long skipped = in().skip(n); + setOffset(offset() + skipped); + return skipped; + } + + protected abstract void initInputStream() throws IOException; + + @Override + public OutputStream out() throws IOException { + return null; + } + + @Override + public boolean isWritable() { + return false; + } + + @Override + public boolean isReadable() { + return true; + } + + @Override + public long length() throws IOException { + return raw().length(); + } + + @Override + public boolean exists() throws IOException { + return raw().exists(); + } + + @Override + public void setLength(long length) throws IOException { + throw new IOException("This handle " + this.getClass().getSimpleName() + + " is read-only!"); + } + + /** + * @return the raw underlying DataHandle (not decompressed) + */ + protected DataHandle raw() { + if (rawHandle == null) { + rawHandle = dataHandleService.create(get().getBaseLocation()); + } + return rawHandle; + } + +} diff --git a/src/main/java/org/scijava/io/location/AbstractHigherOrderLocation.java b/src/main/java/org/scijava/io/location/AbstractHigherOrderLocation.java new file mode 100644 index 000000000..0a8402fc7 --- /dev/null +++ b/src/main/java/org/scijava/io/location/AbstractHigherOrderLocation.java @@ -0,0 +1,52 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck + * Institute of Molecular Cell Biology and Genetics, University of + * Konstanz, and KNIME GmbH. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +/** + * Abstract superclass for {@link Location} implementations that wrap other + * locations. + * + * @author Gabriel Einsdorf + */ +public abstract class AbstractHigherOrderLocation implements Location { + + private final Location baseLocation; + + public AbstractHigherOrderLocation(final Location location) { + this.baseLocation = location; + } + + public Location getBaseLocation() { + return baseLocation; + } +} diff --git a/src/main/java/org/scijava/io/location/BZip2Location.java b/src/main/java/org/scijava/io/location/BZip2Location.java new file mode 100644 index 000000000..1273604ea --- /dev/null +++ b/src/main/java/org/scijava/io/location/BZip2Location.java @@ -0,0 +1,24 @@ + +package org.scijava.io.location; + +import org.scijava.io.handle.BZip2Handle; +import org.scijava.io.handle.DataHandle; + +/** + * {@link Location} backed by a {@link DataHandle} that is BZip2 compressed. + * + * @author Gabriel Einsdorf + * @see BZip2Handle + */ +public class BZip2Location extends AbstractHigherOrderLocation { + + /** + * Creates a {@link BZip2Location} wrapping the given location + * + * @param location the location to operate on + */ + public BZip2Location(final Location location) { + super(location); + } + +} diff --git a/src/main/java/org/scijava/io/location/GZipLocation.java b/src/main/java/org/scijava/io/location/GZipLocation.java new file mode 100644 index 000000000..41a465480 --- /dev/null +++ b/src/main/java/org/scijava/io/location/GZipLocation.java @@ -0,0 +1,20 @@ + +package org.scijava.io.location; + +import org.scijava.io.handle.DataHandle; +import org.scijava.io.handle.GZipHandle; + +/** + * {@link Location} backed by a {@link DataHandle} that is gzip + * compressed. + * + * @author Gabriel Einsdorf + * @see GZipHandle + */ +public class GZipLocation extends AbstractHigherOrderLocation { + + public GZipLocation(Location location) { + super(location); + } + +} diff --git a/src/main/java/org/scijava/io/location/ZipLocation.java b/src/main/java/org/scijava/io/location/ZipLocation.java new file mode 100644 index 000000000..7b050aa78 --- /dev/null +++ b/src/main/java/org/scijava/io/location/ZipLocation.java @@ -0,0 +1,86 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +import java.util.zip.ZipEntry; + +import org.scijava.io.handle.DataHandle; +import org.scijava.io.handle.ZipHandle; +import org.scijava.plugin.Plugin; + +/** + * {@link Location} backed by a {@link DataHandle} that is zip + * compressed. + * + * @author Gabriel Einsdorf + * @see ZipHandle + */ +@Plugin(type = DataHandle.class) +public class ZipLocation extends AbstractHigherOrderLocation { + + private ZipEntry entry; + + public ZipLocation(Location location) { + super(location); + } + + public ZipLocation(Location location, ZipEntry entry) { + super(location); + this.entry = entry; + } + + public ZipEntry getEntry() { + return entry; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((entry == null) ? 0 : entry.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + ZipLocation other = (ZipLocation) obj; + if (entry == null) { + if (other.entry != null) return false; + } + else if (!entry.equals(other.entry)) return false; + return true; + } + +} diff --git a/src/test/java/org/scijava/io/handle/BZip2HandleTest.java b/src/test/java/org/scijava/io/handle/BZip2HandleTest.java new file mode 100644 index 000000000..66138c663 --- /dev/null +++ b/src/test/java/org/scijava/io/handle/BZip2HandleTest.java @@ -0,0 +1,97 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.junit.Ignore; +import org.junit.Test; +import org.scijava.io.handle.DataHandle; +import org.scijava.io.location.BZip2Location; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.Location; + +/** + * Tests {@link BZip2Handle}. + * + * @author Gabriel Einsdorf + */ +public class BZip2HandleTest extends DataHandleTest { + + @Override + public Class> getExpectedHandleType() { + return BZip2Handle.class; + } + + @Override + public Location createLocation() throws IOException { + // create and populate a temp file + final File tmpFile = File.createTempFile("FileHandleTest", "test-file"); + tmpFile.deleteOnExit(); + + try (FileOutputStream out = new FileOutputStream(tmpFile)) { + populateData(out); + } + + final Runtime rt = Runtime.getRuntime(); + final Process p = rt.exec(new String[] { "bzip2", tmpFile + .getAbsolutePath() }); + try { + p.waitFor(); + } + catch (final InterruptedException e) { + throw new RuntimeException(e); + } + final File bzip2File = new File(tmpFile.getAbsolutePath() + ".bz2"); + bzip2File.deleteOnExit(); + + return new BZip2Location(new FileLocation(bzip2File)); + } + + @Test + @Override + public void testWriting() throws IOException { + // no Op + } + + @Override + @Test + public void testReading() throws IOException { + try (final DataHandle handle = createHandle()) { + checkBasicReadMethods(handle, false); + checkEndiannessReading(handle); + } + } + +} diff --git a/src/test/java/org/scijava/io/handle/GZipHandleTest.java b/src/test/java/org/scijava/io/handle/GZipHandleTest.java new file mode 100644 index 000000000..9575960ad --- /dev/null +++ b/src/test/java/org/scijava/io/handle/GZipHandleTest.java @@ -0,0 +1,87 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +import org.junit.Ignore; +import org.junit.Test; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.GZipLocation; +import org.scijava.io.location.Location; + +/** + * Tests {@link GZipHandle}. + * + * @author Gabriel Einsdorf + */ +public class GZipHandleTest extends DataHandleTest { + + @Override + public Class> getExpectedHandleType() { + return GZipHandle.class; + } + + @Override + public Location createLocation() throws IOException { + // create and populate a temp file + final File tmpFile = File.createTempFile("FileHandleTest", + "test-file.gzip"); + tmpFile.deleteOnExit(); + + try (GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream( + tmpFile))) + { + populateData(out); + } + + return new GZipLocation(new FileLocation(tmpFile)); + } + + @Test + @Override + public void testWriting() throws IOException { + // no Op + } + + @Override + @Test + public void testReading() throws IOException { + try (final DataHandle handle = createHandle()) { + checkBasicReadMethods(handle, false); + checkEndiannessReading(handle); + } + } +} diff --git a/src/test/java/org/scijava/io/handle/ZipHandleTest.java b/src/test/java/org/scijava/io/handle/ZipHandleTest.java new file mode 100644 index 000000000..b17d37c9f --- /dev/null +++ b/src/test/java/org/scijava/io/handle/ZipHandleTest.java @@ -0,0 +1,89 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.handle; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.junit.Ignore; +import org.junit.Test; +import org.scijava.io.handle.DataHandle; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.Location; +import org.scijava.io.location.ZipLocation; + +/** + * Tests {@link ZipHandle}. + * + * @author Gabriel Einsdorf + */ +public class ZipHandleTest extends DataHandleTest { + + @Override + public Class> getExpectedHandleType() { + return ZipHandle.class; + } + + @Override + public Location createLocation() throws IOException { + // create and populate a temp file + final File tmpFile = File.createTempFile("FileHandleTest", "test-file.zip"); + tmpFile.deleteOnExit(); + + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream( + tmpFile))) + { + out.putNextEntry(new ZipEntry(tmpFile.getName())); + populateData(out); + } + + return new ZipLocation(new FileLocation(tmpFile)); + } + + @Test + @Override + public void testWriting() throws IOException { + // no Op + } + + @Override + @Test + public void testReading() throws IOException { + try (final DataHandle handle = createHandle()) { + checkBasicReadMethods(handle, false); + checkEndiannessReading(handle); + } + } +} diff --git a/src/test/java/org/scijava/io/location/BZip2LocationTest.java b/src/test/java/org/scijava/io/location/BZip2LocationTest.java new file mode 100644 index 000000000..c23cba8e9 --- /dev/null +++ b/src/test/java/org/scijava/io/location/BZip2LocationTest.java @@ -0,0 +1,63 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.junit.Test; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.Location; + +/** + * Tests {@link BZip2Location}. + * + * @author Gabriel Einsdorf + */ +public class BZip2LocationTest { + + /** + * Tests {@link BZip2Location#BZip2Location(Location)}. + * + * @throws IOException + */ + @Test + public void testFile() throws IOException { + final String path = "/not/actually/a/real-file"; + final FileLocation loc = new FileLocation(path); + BZip2Location zLoc = new BZip2Location(loc); + + assertEquals(zLoc.getBaseLocation(), loc); + } + +} diff --git a/src/test/java/org/scijava/io/location/GZipLocationTest.java b/src/test/java/org/scijava/io/location/GZipLocationTest.java new file mode 100644 index 000000000..fe8862131 --- /dev/null +++ b/src/test/java/org/scijava/io/location/GZipLocationTest.java @@ -0,0 +1,63 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.junit.Test; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.Location; + +/** + * Tests {@link GZipLocation}. + * + * @author Gabriel Einsdorf + */ +public class GZipLocationTest { + + /** + * Tests {@link GZipLocation#GZipLocation(Location)}. + * + * @throws IOException + */ + @Test + public void testFile() throws IOException { + final String path = "/not/actually/a/real-file"; + final FileLocation loc = new FileLocation(path); + GZipLocation zLoc = new GZipLocation(loc); + + assertEquals(zLoc.getBaseLocation(), loc); + } + +} diff --git a/src/test/java/org/scijava/io/location/ZipLocationTest.java b/src/test/java/org/scijava/io/location/ZipLocationTest.java new file mode 100644 index 000000000..a171d6cc5 --- /dev/null +++ b/src/test/java/org/scijava/io/location/ZipLocationTest.java @@ -0,0 +1,63 @@ +/* + * #%L + * SciJava Common shared library for SciJava software. + * %% + * Copyright (C) 2009 - 2017 Board of Regents of the University of + * Wisconsin-Madison, Broad Institute of MIT and Harvard, and Max Planck + * Institute of Molecular Cell Biology and Genetics. + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package org.scijava.io.location; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.junit.Test; +import org.scijava.io.location.FileLocation; +import org.scijava.io.location.Location; + +/** + * Tests {@link ZipLocation}. + * + * @author Gabriel Einsdorf + */ +public class ZipLocationTest { + + /** + * Tests {@link ZipLocation#ZipLocation(Location)}. + * + * @throws IOException + */ + @Test + public void testFile() throws IOException { + final String path = "/not/actually/a/real-file"; + final FileLocation loc = new FileLocation(path); + ZipLocation zLoc = new ZipLocation(loc); + + assertEquals(zLoc.getBaseLocation(), loc); + } + +}