From 9e907515d8babd5bcdeeff405357e15f6b9dd94a Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sat, 3 Apr 2021 01:51:11 -0400 Subject: [PATCH 01/10] Added Tensor.dataToString(). --- .../src/main/java/org/tensorflow/Tensor.java | 46 ++++++ .../tensorflow/internal/types/Tensors.java | 136 ++++++++++++++++++ .../test/java/org/tensorflow/TensorTest.java | 35 ++++- 3 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java index fc1275229bf..5934a47c85d 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java @@ -191,6 +191,33 @@ static T of(Class type, Shape shape, ByteDataBuffer rawData */ long numBytes(); + /** + * Returns the String representation of elements stored in the tensor. + * + * @param options overrides the default configuration + * @return the String representation of the tensor + * @throws IllegalStateException if this is an operand of a graph + */ + default String dataToString(ToStringOptions... options) { + Integer maxWidth = null; + if (options != null) { + for (ToStringOptions opts : options) { + if (opts.maxWidth != null) { + maxWidth = opts.maxWidth; + } + } + } + return Tensors.toString(this, maxWidth); + } + + /** + * @param maxWidth the maximum width of the output ({@code null} if unlimited). This limit may + * surpassed if the first or last element are too long. + */ + public static ToStringOptions maxWidth(Integer maxWidth) { + return new ToStringOptions().maxWidth(maxWidth); + } + /** * Returns the shape of the tensor. */ @@ -212,4 +239,23 @@ static T of(Class type, Shape shape, ByteDataBuffer rawData */ @Override void close(); + + public static class ToStringOptions { + + /** + * Sets the maximum width of the output. + * + * @param maxWidth the maximum width of the output ({@code null} if unlimited). This limit may + * surpassed if the first or last element are too long. + */ + public ToStringOptions maxWidth(Integer maxWidth) { + this.maxWidth = maxWidth; + return this; + } + + private Integer maxWidth; + + private ToStringOptions() { + } + } } diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java new file mode 100644 index 00000000000..65109893ad5 --- /dev/null +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java @@ -0,0 +1,136 @@ +package org.tensorflow.internal.types; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.StringJoiner; +import org.tensorflow.Tensor; +import org.tensorflow.ndarray.NdArray; +import org.tensorflow.ndarray.Shape; + +/** + * Tensor helper methods. + */ +public final class Tensors { + + /** + * Prevent construction. + */ + private Tensors() { + } + + /** + * @param tensor a tensor + * @param maxWidth the maximum width of the output ({@code null} if absent) + * @return the String representation of the tensor + */ + public static String toString(Tensor tensor, Integer maxWidth) { + if (!(tensor instanceof NdArray)) { + throw new AssertionError("Expected tensor to extend NdArray"); + } + NdArray ndArray = (NdArray) tensor; + Iterator> iterator = ndArray.scalars().iterator(); + Shape shape = tensor.shape(); + if (shape.numDimensions() == 0) { + if (!iterator.hasNext()) { + return ""; + } + return String.valueOf(iterator.next().getObject()); + } + return toString(iterator, shape, 0, maxWidth); + } + + /** + * @param iterator an iterator over the scalars + * @param shape the shape of the tensor + * @param maxWidth the maximum width of the output ({@code null} if absent) + * @param dimension the current dimension being processed + * @return the String representation of the tensor data at {@code dimension} + */ + private static String toString(Iterator> iterator, Shape shape, + int dimension, Integer maxWidth) { + if (dimension < shape.numDimensions() - 1) { + StringJoiner joiner = new StringJoiner(",\n", indent(dimension) + "[\n", + "\n" + indent(dimension) + "]"); + for (long i = 0, size = shape.size(dimension); i < size; ++i) { + String element = toString(iterator, shape, dimension + 1, maxWidth); + joiner.add(element); + } + return joiner.toString(); + } + if (maxWidth == null) { + StringJoiner joiner = new StringJoiner(", ", indent(dimension) + "[", "]"); + for (long i = 0, size = shape.size(dimension); i < size; ++i) { + String element = iterator.next().getObject().toString(); + joiner.add(element); + } + return joiner.toString(); + } + List lengths = new ArrayList<>(); + StringJoiner joiner = new StringJoiner(", ", indent(dimension) + "[", "]"); + int lengthBefore = joiner.length() - "]".length(); + for (long i = 0, size = shape.size(dimension); i < size; ++i) { + String element = iterator.next().getObject().toString(); + joiner.add(element); + int addedLength = joiner.length() - lengthBefore; + lengths.add(addedLength); + lengthBefore += addedLength; + } + if (joiner.length() <= maxWidth) { + return joiner.toString(); + } + StringBuilder result = new StringBuilder(joiner.toString()); + int midPoint = (maxWidth / 2) - 1; + int width = 0; + int indexOfElementToRemove = lengths.size() - 1; + int widthBeforeElementToRemove = 0; + for (int i = 0, size = lengths.size(); i < size; ++i) { + width += lengths.get(i); + if (width > midPoint) { + indexOfElementToRemove = i; + break; + } + widthBeforeElementToRemove = width; + } + if (indexOfElementToRemove == 0) { + // Cannot remove first element + return joiner.toString(); + } + result.insert(widthBeforeElementToRemove, ", ..."); + widthBeforeElementToRemove += ", ...".length(); + width = result.length(); + while (width > maxWidth) { + if (indexOfElementToRemove == 0) { + // Cannot remove first element + break; + } else if (indexOfElementToRemove == lengths.size() - 1) { + // Cannot remove last element + --indexOfElementToRemove; + continue; + } + Integer length = lengths.remove(indexOfElementToRemove); + result.delete(widthBeforeElementToRemove, widthBeforeElementToRemove + length); + width = result.length(); + } + if (result.length() < joiner.length()) { + return result.toString(); + } + // Do not insert ellipses if it increases the length + return joiner.toString(); + } + + /** + * @param level the level of indent + * @return the indentation string + */ + public static String indent(int level) { + if (level <= 0) { + return ""; + } + StringBuilder result = new StringBuilder(level * 2); + for (int i = 0; i < level; ++i) { + result.append(" "); + } + return result.toString(); + } +} \ No newline at end of file diff --git a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java index 9415a986222..57f52a94e40 100644 --- a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java @@ -340,7 +340,7 @@ public void nDimensional() { } LongNdArray threeD = StdArrays.ndCopyOf(new long[][][]{ - {{1}, {3}, {5}, {7}, {9}}, {{2}, {4}, {6}, {8}, {0}}, + {{1}, {3}, {5}, {7}, {9}}, {{2}, {4}, {6}, {8}, {0}}, }); try (TInt64 t = TInt64.tensorOf(threeD)) { assertEquals(TInt64.class, t.type()); @@ -353,9 +353,9 @@ public void nDimensional() { } BooleanNdArray fourD = StdArrays.ndCopyOf(new boolean[][][][]{ - {{{false, false, false, true}, {false, false, true, false}}}, - {{{false, false, true, true}, {false, true, false, false}}}, - {{{false, true, false, true}, {false, true, true, false}}}, + {{{false, false, false, true}, {false, false, true, false}}}, + {{{false, false, true, true}, {false, true, false, false}}}, + {{{false, true, false, true}, {false, true, true, false}}}, }); try (TBool t = TBool.tensorOf(fourD)) { assertEquals(TBool.class, t.type()); @@ -541,6 +541,33 @@ public void gracefullyFailCreationFromNullArrayForStringTensor() { } } + @Test + public void dataToString() { + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + String actual = t.dataToString(); + assertEquals("[3, 0, 1]", actual); + } + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + String actual = t.dataToString(Tensor.maxWidth(5)); + // Cannot remove first or last element + assertEquals("[3, 0, 1]", actual); + } + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + String actual = t.dataToString(Tensor.maxWidth(6)); + // Do not insert ellipses if it increases the length + assertEquals("[3, 0, 1]", actual); + } + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2}))) { + String actual = t.dataToString(Tensor.maxWidth(11)); + // Limit may be surpassed if first or last element are too long + assertEquals("[3, ..., 2]", actual); + } + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2}))) { + String actual = t.dataToString(Tensor.maxWidth(12)); + assertEquals("[3, 0, 1, 2]", actual); + } + } + // Workaround for cross compiliation // (e.g., javac -source 1.9 -target 1.8). // From b3a2ab746b32511c9a4c36c9c4f3ce28540b24cb Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sat, 3 Apr 2021 01:56:11 -0400 Subject: [PATCH 02/10] Added missing import. --- .../tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java index 5934a47c85d..b58f2f04216 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java @@ -16,6 +16,7 @@ package org.tensorflow; import java.util.function.Consumer; +import org.tensorflow.internal.types.Tensors; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.Shaped; import org.tensorflow.ndarray.buffer.ByteDataBuffer; From e30d9b52fa3fc4ac9fb0bbd311c9343521963e76 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sat, 3 Apr 2021 10:22:35 -0400 Subject: [PATCH 03/10] Documentation fix. --- .../src/main/java/org/tensorflow/internal/types/Tensors.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java index 65109893ad5..5160d1284f8 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java @@ -21,7 +21,7 @@ private Tensors() { /** * @param tensor a tensor - * @param maxWidth the maximum width of the output ({@code null} if absent) + * @param maxWidth the maximum width of the output ({@code null} if unlimited) * @return the String representation of the tensor */ public static String toString(Tensor tensor, Integer maxWidth) { @@ -43,7 +43,7 @@ public static String toString(Tensor tensor, Integer maxWidth) { /** * @param iterator an iterator over the scalars * @param shape the shape of the tensor - * @param maxWidth the maximum width of the output ({@code null} if absent) + * @param maxWidth the maximum width of the output ({@code null} if unlimited) * @param dimension the current dimension being processed * @return the String representation of the tensor data at {@code dimension} */ From 8bf627a9b85ff2ad978ab2625ccfdd0c91e5e334 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sat, 3 Apr 2021 10:27:10 -0400 Subject: [PATCH 04/10] Documentation fix. --- .../src/main/java/org/tensorflow/Tensor.java | 114 +++++++++--------- .../tensorflow/internal/types/Tensors.java | 6 +- 2 files changed, 64 insertions(+), 56 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java index b58f2f04216..7b5f3d1b439 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java @@ -27,9 +27,9 @@ * A statically typed multi-dimensional array. * *

There are two categories of tensors in TensorFlow Java: {@link TType typed tensors} and - * {@link RawTensor raw tensors}. The former maps the tensor native memory to an - * n-dimensional typed data space, allowing direct I/O operations from the JVM, while the latter - * is only a reference to a native tensor allowing basic operations and flat data access.

+ * {@link RawTensor raw tensors}. The former maps the tensor native memory to an n-dimensional typed + * data space, allowing direct I/O operations from the JVM, while the latter is only a reference to + * a native tensor allowing basic operations and flat data access.

* *

WARNING: Resources consumed by the Tensor object must be explicitly freed by * invoking the {@link #close()} method when the object is no longer needed. For example, using a @@ -50,15 +50,15 @@ public interface Tensor extends Shaped, AutoCloseable { *

The amount of memory to allocate is derived from the datatype and the shape of the tensor, * and is left uninitialized. * - * @param the tensor type - * @param type the tensor type class + * @param the tensor type + * @param type the tensor type class * @param shape shape of the tensor * @return an allocated but uninitialized tensor * @throws IllegalArgumentException if elements of the given {@code type} are of variable length * (e.g. strings) - * @throws IllegalArgumentException if {@code shape} is totally or partially - * {@link Shape#hasUnknownDimension() unknown} - * @throws IllegalStateException if tensor failed to be allocated + * @throws IllegalArgumentException if {@code shape} is totally or partially {@link + * Shape#hasUnknownDimension() unknown} + * @throws IllegalStateException if tensor failed to be allocated */ static T of(Class type, Shape shape) { return of(type, shape, -1); @@ -68,27 +68,27 @@ static T of(Class type, Shape shape) { * Allocates a tensor of a given datatype, shape and size. * *

This method is identical to {@link #of(Class, Shape)}, except that the final size of the - * tensor can be explicitly set instead of computing it from the datatype and shape, which could be - * larger than the actual space required to store the data but not smaller. + * tensor can be explicitly set instead of computing it from the datatype and shape, which could + * be larger than the actual space required to store the data but not smaller. * - * @param the tensor type - * @param type the tensor type class + * @param the tensor type + * @param type the tensor type class * @param shape shape of the tensor - * @param size size in bytes of the tensor or -1 to compute the size from the shape + * @param size size in bytes of the tensor or -1 to compute the size from the shape * @return an allocated but uninitialized tensor - * @see #of(Class, Shape) * @throws IllegalArgumentException if {@code size} is smaller than the minimum space required to * store the tensor data - * @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given - * {@code type} are of variable length (e.g. strings) - * @throws IllegalArgumentException if {@code shape} is totally or partially - * {@link Shape#hasUnknownDimension() unknown} - * @throws IllegalStateException if tensor failed to be allocated + * @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given {@code + * type} are of variable length (e.g. strings) + * @throws IllegalArgumentException if {@code shape} is totally or partially {@link + * Shape#hasUnknownDimension() unknown} + * @throws IllegalStateException if tensor failed to be allocated + * @see #of(Class, Shape) */ static T of(Class type, Shape shape, long size) { RawTensor tensor = RawTensor.allocate(type, shape, size); try { - return (T)tensor.asTypedTensor(); + return (T) tensor.asTypedTensor(); } catch (Exception e) { tensor.close(); throw e; @@ -112,16 +112,17 @@ static T of(Class type, Shape shape, long size) { *

If {@code dataInitializer} fails and throws an exception, the allocated tensor will be * automatically released before rethrowing the same exception. * - * @param the tensor type - * @param type the tensor type class - * @param shape shape of the tensor - * @param dataInitializer method receiving accessor to the allocated tensor data for initialization + * @param the tensor type + * @param type the tensor type class + * @param shape shape of the tensor + * @param dataInitializer method receiving accessor to the allocated tensor data for + * initialization * @return an allocated and initialized tensor * @throws IllegalArgumentException if elements of the given {@code type} are of variable length * (e.g. strings) - * @throws IllegalArgumentException if {@code shape} is totally or partially - * {@link Shape#hasUnknownDimension() unknown} - * @throws IllegalStateException if tensor failed to be allocated + * @throws IllegalArgumentException if {@code shape} is totally or partially {@link + * Shape#hasUnknownDimension() unknown} + * @throws IllegalStateException if tensor failed to be allocated */ static T of(Class type, Shape shape, Consumer dataInitializer) { return of(type, shape, -1, dataInitializer); @@ -131,27 +132,31 @@ static T of(Class type, Shape shape, Consumer dataInitia * Allocates a tensor of a given datatype, shape and size. * *

This method is identical to {@link #of(Class, Shape, Consumer)}, except that the final - * size for the tensor can be explicitly set instead of being computed from the datatype and shape. + * size for the tensor can be explicitly set instead of being computed from the datatype and + * shape. * - *

This could be useful for tensor types that stores data but also metadata in the tensor memory, + *

This could be useful for tensor types that stores data but also metadata in the tensor + * memory, * such as the lookup table in a tensor of strings. * - * @param the tensor type - * @param type the tensor type class - * @param shape shape of the tensor - * @param size size in bytes of the tensor or -1 to compute the size from the shape - * @param dataInitializer method receiving accessor to the allocated tensor data for initialization + * @param the tensor type + * @param type the tensor type class + * @param shape shape of the tensor + * @param size size in bytes of the tensor or -1 to compute the size from the shape + * @param dataInitializer method receiving accessor to the allocated tensor data for + * initialization * @return an allocated and initialized tensor - * @see #of(Class, Shape, long, Consumer) * @throws IllegalArgumentException if {@code size} is smaller than the minimum space required to * store the tensor data - * @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given - * {@code type} are of variable length (e.g. strings) - * @throws IllegalArgumentException if {@code shape} is totally or partially - * {@link Shape#hasUnknownDimension() unknown} - * @throws IllegalStateException if tensor failed to be allocated + * @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given {@code + * type} are of variable length (e.g. strings) + * @throws IllegalArgumentException if {@code shape} is totally or partially {@link + * Shape#hasUnknownDimension() unknown} + * @throws IllegalStateException if tensor failed to be allocated + * @see #of(Class, Shape, long, Consumer) */ - static T of(Class type, Shape shape, long size, Consumer dataInitializer) { + static T of(Class type, Shape shape, long size, + Consumer dataInitializer) { T tensor = of(type, shape, size); try { dataInitializer.accept(tensor); @@ -168,18 +173,19 @@ static T of(Class type, Shape shape, long size, Consumer *

Data must have been encoded into {@code data} as per the specification of the TensorFlow C API. * - * @param the tensor type - * @param type the tensor type class - * @param shape the tensor shape. + * @param the tensor type + * @param type the tensor type class + * @param shape the tensor shape. * @param rawData a buffer containing the tensor raw data. * @throws IllegalArgumentException if {@code rawData} is not large enough to contain the tensor * data - * @throws IllegalArgumentException if {@code shape} is totally or partially - * {@link Shape#hasUnknownDimension() unknown} - * @throws IllegalStateException if tensor failed to be allocated with the given parameters + * @throws IllegalArgumentException if {@code shape} is totally or partially {@link + * Shape#hasUnknownDimension() unknown} + * @throws IllegalStateException if tensor failed to be allocated with the given parameters */ static T of(Class type, Shape shape, ByteDataBuffer rawData) { - return of(type, shape, rawData.size(), t -> rawData.copyTo(t.asRawTensor().data(), rawData.size())); + return of(type, shape, rawData.size(), + t -> rawData.copyTo(t.asRawTensor().data(), rawData.size())); } /** @@ -212,10 +218,10 @@ default String dataToString(ToStringOptions... options) { } /** - * @param maxWidth the maximum width of the output ({@code null} if unlimited). This limit may - * surpassed if the first or last element are too long. + * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). This + * limit may surpassed if the first or last element are too long. */ - public static ToStringOptions maxWidth(Integer maxWidth) { + static ToStringOptions maxWidth(Integer maxWidth) { return new ToStringOptions().maxWidth(maxWidth); } @@ -244,10 +250,10 @@ public static ToStringOptions maxWidth(Integer maxWidth) { public static class ToStringOptions { /** - * Sets the maximum width of the output. + * Sets the maximum width of the output in characters. * - * @param maxWidth the maximum width of the output ({@code null} if unlimited). This limit may - * surpassed if the first or last element are too long. + * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). + * This limit may surpassed if the first or last element are too long. */ public ToStringOptions maxWidth(Integer maxWidth) { this.maxWidth = maxWidth; diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java index 5160d1284f8..cceb96f20ff 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java @@ -21,7 +21,8 @@ private Tensors() { /** * @param tensor a tensor - * @param maxWidth the maximum width of the output ({@code null} if unlimited) + * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). This + * limit may surpassed if the first or last element are too long. * @return the String representation of the tensor */ public static String toString(Tensor tensor, Integer maxWidth) { @@ -43,7 +44,8 @@ public static String toString(Tensor tensor, Integer maxWidth) { /** * @param iterator an iterator over the scalars * @param shape the shape of the tensor - * @param maxWidth the maximum width of the output ({@code null} if unlimited) + * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). + * This limit may surpassed if the first or last element are too long. * @param dimension the current dimension being processed * @return the String representation of the tensor data at {@code dimension} */ From 61e7cad2eb85e549ed7c49af587406b482f93cf5 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sat, 3 Apr 2021 10:47:15 -0400 Subject: [PATCH 05/10] maxWidth was truncating text mid-element. --- .../tensorflow/internal/types/Tensors.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java index cceb96f20ff..77bc35a042d 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java @@ -19,6 +19,16 @@ public final class Tensors { private Tensors() { } + /** + * Equivalent to {@link #toString(Tensor, Integer) toString(tensor, null)}. + * + * @param tensor a tensor + * @return the String representation of the tensor + */ + public static String toString(Tensor tensor) { + return toString(tensor, null); + } + /** * @param tensor a tensor * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). This @@ -70,7 +80,7 @@ private static String toString(Iterator> iterator, Shape sh } List lengths = new ArrayList<>(); StringJoiner joiner = new StringJoiner(", ", indent(dimension) + "[", "]"); - int lengthBefore = joiner.length() - "]".length(); + int lengthBefore = "]".length(); for (long i = 0, size = shape.size(dimension); i < size; ++i) { String element = iterator.next().getObject().toString(); joiner.add(element); @@ -78,10 +88,20 @@ private static String toString(Iterator> iterator, Shape sh lengths.add(addedLength); lengthBefore += addedLength; } - if (joiner.length() <= maxWidth) { - return joiner.toString(); + return truncateWidth(joiner.toString(), maxWidth, lengths); + } + + /** + * @param input the input to truncate + * @param maxWidth the maximum width of the output in characters + * @param lengths the lengths of elements inside input + * @return the (potentially) truncated output + */ + private static String truncateWidth(String input, int maxWidth, List lengths) { + if (input.length() <= maxWidth) { + return input; } - StringBuilder result = new StringBuilder(joiner.toString()); + StringBuilder output = new StringBuilder(input); int midPoint = (maxWidth / 2) - 1; int width = 0; int indexOfElementToRemove = lengths.size() - 1; @@ -96,11 +116,11 @@ private static String toString(Iterator> iterator, Shape sh } if (indexOfElementToRemove == 0) { // Cannot remove first element - return joiner.toString(); + return input; } - result.insert(widthBeforeElementToRemove, ", ..."); + output.insert(widthBeforeElementToRemove, ", ..."); widthBeforeElementToRemove += ", ...".length(); - width = result.length(); + width = output.length(); while (width > maxWidth) { if (indexOfElementToRemove == 0) { // Cannot remove first element @@ -111,14 +131,14 @@ private static String toString(Iterator> iterator, Shape sh continue; } Integer length = lengths.remove(indexOfElementToRemove); - result.delete(widthBeforeElementToRemove, widthBeforeElementToRemove + length); - width = result.length(); + output.delete(widthBeforeElementToRemove, widthBeforeElementToRemove + length); + width = output.length(); } - if (result.length() < joiner.length()) { - return result.toString(); + if (output.length() < input.length()) { + return output.toString(); } // Do not insert ellipses if it increases the length - return joiner.toString(); + return input; } /** From ffdf8794fe8257f9bf6a05a30a37b563f147e5f5 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sun, 4 Apr 2021 14:56:22 -0400 Subject: [PATCH 06/10] * Added Operand.dataToString(). * Added support for Tensors.toString(RawTensor). * Test multidimensional tensor, RawTensor. --- .../src/main/java/org/tensorflow/Operand.java | 12 ++++++++++++ .../src/main/java/org/tensorflow/Tensor.java | 6 ++---- .../tensorflow/{internal/types => }/Tensors.java | 15 +++++++++++++-- .../src/test/java/org/tensorflow/TensorTest.java | 14 +++++++++++++- 4 files changed, 40 insertions(+), 7 deletions(-) rename tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/{internal/types => }/Tensors.java (90%) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Operand.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Operand.java index 80f62eb5acc..9e903cce7e0 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Operand.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Operand.java @@ -15,6 +15,7 @@ package org.tensorflow; +import org.tensorflow.Tensor.ToStringOptions; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.Shaped; import org.tensorflow.op.Op; @@ -65,6 +66,17 @@ default T asTensor() { return asOutput().asTensor(); } + /** + * Returns the String representation of the tensor elements at this operand. + * + * @param options overrides the default configuration + * @return the String representation of the tensor elements + * @throws IllegalStateException if this is an operand of a graph + */ + default String dataToString(ToStringOptions... options) { + return asTensor().dataToString(options); + } + /** * Returns the tensor type of this operand */ diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java index 7b5f3d1b439..bffc0dd9cb6 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java @@ -16,7 +16,6 @@ package org.tensorflow; import java.util.function.Consumer; -import org.tensorflow.internal.types.Tensors; import org.tensorflow.ndarray.Shape; import org.tensorflow.ndarray.Shaped; import org.tensorflow.ndarray.buffer.ByteDataBuffer; @@ -136,8 +135,7 @@ static T of(Class type, Shape shape, Consumer dataInitia * shape. * *

This could be useful for tensor types that stores data but also metadata in the tensor - * memory, - * such as the lookup table in a tensor of strings. + * memory, such as the lookup table in a tensor of strings. * * @param the tensor type * @param type the tensor type class @@ -202,7 +200,7 @@ static T of(Class type, Shape shape, ByteDataBuffer rawData * Returns the String representation of elements stored in the tensor. * * @param options overrides the default configuration - * @return the String representation of the tensor + * @return the String representation of the tensor elements * @throws IllegalStateException if this is an operand of a graph */ default String dataToString(ToStringOptions... options) { diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java similarity index 90% rename from tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java rename to tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java index 77bc35a042d..63982eb81df 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/internal/types/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java @@ -1,4 +1,4 @@ -package org.tensorflow.internal.types; +package org.tensorflow; import java.util.ArrayList; import java.util.Iterator; @@ -11,7 +11,7 @@ /** * Tensor helper methods. */ -public final class Tensors { +final class Tensors { /** * Prevent construction. @@ -30,12 +30,20 @@ public static String toString(Tensor tensor) { } /** + * Returns a String representation of a tensor's data. If the output is wider than {@code + * maxWidth} characters, it is truncated and "{@code ...}" is inserted in place of the removed + * data. + * * @param tensor a tensor * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). This * limit may surpassed if the first or last element are too long. * @return the String representation of the tensor */ public static String toString(Tensor tensor, Integer maxWidth) { + if (tensor instanceof RawTensor) { + System.out.println("Got rawTensor: " + tensor); + tensor = ((RawTensor) tensor).asTypedTensor(); + } if (!(tensor instanceof NdArray)) { throw new AssertionError("Expected tensor to extend NdArray"); } @@ -92,6 +100,9 @@ private static String toString(Iterator> iterator, Shape sh } /** + * Truncates the width of a String if it's too long, inserting "{@code ...}" in place of the + * removed data. + * * @param input the input to truncate * @param maxWidth the maximum width of the output in characters * @param lengths the lengths of elements inside input diff --git a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java index 57f52a94e40..5a8f2ed772f 100644 --- a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java @@ -514,7 +514,8 @@ public void fromHandle() { // close() on both Tensors. final FloatNdArray matrix = StdArrays.ndCopyOf(new float[][]{{1, 2, 3}, {4, 5, 6}}); try (TFloat32 src = TFloat32.tensorOf(matrix)) { - TFloat32 cpy = (TFloat32)RawTensor.fromHandle(src.asRawTensor().nativeHandle()).asTypedTensor(); + TFloat32 cpy = (TFloat32) RawTensor.fromHandle(src.asRawTensor().nativeHandle()) + .asTypedTensor(); assertEquals(src.type(), cpy.type()); assertEquals(src.dataType(), cpy.dataType()); assertEquals(src.shape().numDimensions(), cpy.shape().numDimensions()); @@ -566,6 +567,17 @@ public void dataToString() { String actual = t.dataToString(Tensor.maxWidth(12)); assertEquals("[3, 0, 1, 2]", actual); } + try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[][]{{1, 2, 3}, {3, 2, 1}}))) { + String actual = t.dataToString(Tensor.maxWidth(12)); + assertEquals("[\n" + + " [1, 2, 3]\n" + + " [3, 2, 1]\n" + + "]", actual); + } + try (RawTensor t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2})).asRawTensor()) { + String actual = t.dataToString(Tensor.maxWidth(12)); + assertEquals("[3, 0, 1, 2]", actual); + } } // Workaround for cross compiliation From da04cd47b4d6ed012fbfd95724658472d0780471 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sun, 4 Apr 2021 14:57:54 -0400 Subject: [PATCH 07/10] Fixed test. --- .../src/test/java/org/tensorflow/TensorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java index 5a8f2ed772f..681a1ddc91c 100644 --- a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java @@ -570,7 +570,7 @@ public void dataToString() { try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[][]{{1, 2, 3}, {3, 2, 1}}))) { String actual = t.dataToString(Tensor.maxWidth(12)); assertEquals("[\n" - + " [1, 2, 3]\n" + + " [1, 2, 3],\n" + " [3, 2, 1]\n" + "]", actual); } From 5821e996630b8bfe728af33f7fc1b4d181861364 Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Sun, 4 Apr 2021 15:01:35 -0400 Subject: [PATCH 08/10] Leaving out comma after closing brackets. --- .../src/main/java/org/tensorflow/Tensors.java | 2 +- .../src/test/java/org/tensorflow/TensorTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java index 63982eb81df..2105845de07 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java @@ -70,7 +70,7 @@ public static String toString(Tensor tensor, Integer maxWidth) { private static String toString(Iterator> iterator, Shape shape, int dimension, Integer maxWidth) { if (dimension < shape.numDimensions() - 1) { - StringJoiner joiner = new StringJoiner(",\n", indent(dimension) + "[\n", + StringJoiner joiner = new StringJoiner("\n", indent(dimension) + "[\n", "\n" + indent(dimension) + "]"); for (long i = 0, size = shape.size(dimension); i < size; ++i) { String element = toString(iterator, shape, dimension + 1, maxWidth); diff --git a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java index 681a1ddc91c..5a8f2ed772f 100644 --- a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java @@ -570,7 +570,7 @@ public void dataToString() { try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[][]{{1, 2, 3}, {3, 2, 1}}))) { String actual = t.dataToString(Tensor.maxWidth(12)); assertEquals("[\n" - + " [1, 2, 3],\n" + + " [1, 2, 3]\n" + " [3, 2, 1]\n" + "]", actual); } From 6f8dedcbba8ec7b951d100291fe3e228f6bf43ba Mon Sep 17 00:00:00 2001 From: Ryan Nett Date: Sun, 4 Apr 2021 17:24:02 -0700 Subject: [PATCH 09/10] Data type tests, wrap strings in quotes Signed-off-by: Ryan Nett --- .../src/main/java/org/tensorflow/Tensors.java | 39 +++++++++++++------ .../test/java/org/tensorflow/TensorTest.java | 29 +++++++++++--- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java index 2105845de07..0f91cbfe64b 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java @@ -4,9 +4,9 @@ import java.util.Iterator; import java.util.List; import java.util.StringJoiner; -import org.tensorflow.Tensor; import org.tensorflow.ndarray.NdArray; import org.tensorflow.ndarray.Shape; +import org.tensorflow.proto.framework.DataType; /** * Tensor helper methods. @@ -56,24 +56,39 @@ public static String toString(Tensor tensor, Integer maxWidth) { } return String.valueOf(iterator.next().getObject()); } - return toString(iterator, shape, 0, maxWidth); + return toString(iterator, tensor.dataType(), shape, 0, maxWidth); } /** - * @param iterator an iterator over the scalars - * @param shape the shape of the tensor - * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). - * This limit may surpassed if the first or last element are too long. + * Convert an element of a tensor to string, in a way that may depend on the data type. + * + * @param dtype the tensor's data type + * @param data the element + * @return the element's string representation + */ + private static String elementToString(DataType dtype, Object data) { + if (dtype == DataType.DT_STRING) { + return '"' + data.toString() + '"'; + } else { + return data.toString(); + } + } + + /** + * @param iterator an iterator over the scalars + * @param shape the shape of the tensor + * @param maxWidth the maximum width of the output in characters ({@code null} if unlimited). This limit may surpassed + * if the first or last element are too long. * @param dimension the current dimension being processed * @return the String representation of the tensor data at {@code dimension} */ - private static String toString(Iterator> iterator, Shape shape, + private static String toString(Iterator> iterator, DataType dtype, Shape shape, int dimension, Integer maxWidth) { if (dimension < shape.numDimensions() - 1) { StringJoiner joiner = new StringJoiner("\n", indent(dimension) + "[\n", "\n" + indent(dimension) + "]"); for (long i = 0, size = shape.size(dimension); i < size; ++i) { - String element = toString(iterator, shape, dimension + 1, maxWidth); + String element = toString(iterator, dtype, shape, dimension + 1, maxWidth); joiner.add(element); } return joiner.toString(); @@ -81,8 +96,8 @@ private static String toString(Iterator> iterator, Shape sh if (maxWidth == null) { StringJoiner joiner = new StringJoiner(", ", indent(dimension) + "[", "]"); for (long i = 0, size = shape.size(dimension); i < size; ++i) { - String element = iterator.next().getObject().toString(); - joiner.add(element); + Object element = iterator.next().getObject(); + joiner.add(elementToString(dtype, element)); } return joiner.toString(); } @@ -90,8 +105,8 @@ private static String toString(Iterator> iterator, Shape sh StringJoiner joiner = new StringJoiner(", ", indent(dimension) + "[", "]"); int lengthBefore = "]".length(); for (long i = 0, size = shape.size(dimension); i < size; ++i) { - String element = iterator.next().getObject().toString(); - joiner.add(element); + Object element = iterator.next().getObject(); + joiner.add(elementToString(dtype, element)); int addedLength = joiner.length() - lengthBefore; lengths.add(addedLength); lengthBefore += addedLength; diff --git a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java index 5a8f2ed772f..be816e6f093 100644 --- a/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java +++ b/tensorflow-core/tensorflow-core-api/src/test/java/org/tensorflow/TensorTest.java @@ -544,26 +544,26 @@ public void gracefullyFailCreationFromNullArrayForStringTensor() { @Test public void dataToString() { - try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + try (TInt32 t = TInt32.vectorOf(3, 0, 1)) { String actual = t.dataToString(); assertEquals("[3, 0, 1]", actual); } - try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + try (TInt32 t = TInt32.vectorOf(3, 0, 1)) { String actual = t.dataToString(Tensor.maxWidth(5)); // Cannot remove first or last element assertEquals("[3, 0, 1]", actual); } - try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1}))) { + try (TInt32 t = TInt32.vectorOf(3, 0, 1)) { String actual = t.dataToString(Tensor.maxWidth(6)); // Do not insert ellipses if it increases the length assertEquals("[3, 0, 1]", actual); } - try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2}))) { + try (TInt32 t = TInt32.vectorOf(3, 0, 1, 2)) { String actual = t.dataToString(Tensor.maxWidth(11)); // Limit may be surpassed if first or last element are too long assertEquals("[3, ..., 2]", actual); } - try (TInt32 t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2}))) { + try (TInt32 t = TInt32.vectorOf(3, 0, 1, 2)) { String actual = t.dataToString(Tensor.maxWidth(12)); assertEquals("[3, 0, 1, 2]", actual); } @@ -574,10 +574,27 @@ public void dataToString() { + " [3, 2, 1]\n" + "]", actual); } - try (RawTensor t = TInt32.tensorOf(StdArrays.ndCopyOf(new int[]{3, 0, 1, 2})).asRawTensor()) { + try (RawTensor t = TInt32.vectorOf(3, 0, 1, 2).asRawTensor()) { String actual = t.dataToString(Tensor.maxWidth(12)); assertEquals("[3, 0, 1, 2]", actual); } + // different data types + try (RawTensor t = TFloat32.vectorOf(3.0101f, 0, 1.5f, 2).asRawTensor()) { + String actual = t.dataToString(); + assertEquals("[3.0101, 0.0, 1.5, 2.0]", actual); + } + try (RawTensor t = TFloat64.vectorOf(3.0101, 0, 1.5, 2).asRawTensor()) { + String actual = t.dataToString(); + assertEquals("[3.0101, 0.0, 1.5, 2.0]", actual); + } + try (RawTensor t = TBool.vectorOf(true, true, false, true).asRawTensor()) { + String actual = t.dataToString(); + assertEquals("[true, true, false, true]", actual); + } + try (RawTensor t = TString.vectorOf("a", "b", "c").asRawTensor()) { + String actual = t.dataToString(); + assertEquals("[\"a\", \"b\", \"c\"]", actual); + } } // Workaround for cross compiliation From 98d320da5d0b3a67550016c374a5100930dacdbc Mon Sep 17 00:00:00 2001 From: Gili Tzabari Date: Mon, 5 Apr 2021 22:33:59 -0400 Subject: [PATCH 10/10] Cleanup in response to PR comments. --- .../src/main/java/org/tensorflow/Tensor.java | 2 +- .../src/main/java/org/tensorflow/Tensors.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java index bffc0dd9cb6..87f0f1e7118 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensor.java @@ -245,7 +245,7 @@ static ToStringOptions maxWidth(Integer maxWidth) { @Override void close(); - public static class ToStringOptions { + class ToStringOptions { /** * Sets the maximum width of the output in characters. diff --git a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java index 2105845de07..6b449df9c2d 100644 --- a/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java +++ b/tensorflow-core/tensorflow-core-api/src/main/java/org/tensorflow/Tensors.java @@ -41,11 +41,13 @@ public static String toString(Tensor tensor) { */ public static String toString(Tensor tensor, Integer maxWidth) { if (tensor instanceof RawTensor) { - System.out.println("Got rawTensor: " + tensor); tensor = ((RawTensor) tensor).asTypedTensor(); } if (!(tensor instanceof NdArray)) { - throw new AssertionError("Expected tensor to extend NdArray"); + throw new AssertionError("Expected tensor to extend NdArray.\n" + + "actual : " + tensor + "\n" + + "dataType: " + tensor.dataType() + "\n" + + "class : " + tensor.getClass()); } NdArray ndArray = (NdArray) tensor; Iterator> iterator = ndArray.scalars().iterator();