diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultExpressionHandler.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultExpressionHandler.java index 446fd5cf24..565943308e 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultExpressionHandler.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultExpressionHandler.java @@ -48,40 +48,12 @@ import static io.delta.kernel.defaults.internal.DefaultKernelUtils.daysSinceEpoch; public class DefaultExpressionHandler - implements ExpressionHandler -{ + implements ExpressionHandler { @Override public ExpressionEvaluator getEvaluator(StructType batchSchema, Expression expression) { return new DefaultExpressionEvaluator(expression); } - private static class DefaultExpressionEvaluator - implements ExpressionEvaluator { - private final Expression expression; - - private DefaultExpressionEvaluator(Expression expression) { - this.expression = expression; - } - - @Override - public ColumnVector eval(ColumnarBatch input) { - if (expression instanceof Literal) { - return evalLiteralExpression(input, (Literal) expression); - } - - if (expression.dataType().equals(BooleanType.INSTANCE)) { - return evalBooleanOutputExpression(input, expression); - } - // TODO: Boolean output type expressions are good enough for first preview release - // which enables partition pruning and file skipping using file stats. - - throw new UnsupportedOperationException("not yet implemented"); - } - - @Override - public void close() { /* nothing to close */ } - } - private static ColumnVector evalBooleanOutputExpression( ColumnarBatch input, Expression expression) { checkArgument(expression.dataType().equals(BooleanType.INSTANCE), @@ -133,4 +105,31 @@ private static ColumnVector evalLiteralExpression(ColumnarBatch input, Literal l throw new UnsupportedOperationException( "unsupported expression encountered: " + literal); } + + private static class DefaultExpressionEvaluator + implements ExpressionEvaluator { + private final Expression expression; + + private DefaultExpressionEvaluator(Expression expression) { + this.expression = expression; + } + + @Override + public ColumnVector eval(ColumnarBatch input) { + if (expression instanceof Literal) { + return evalLiteralExpression(input, (Literal) expression); + } + + if (expression.dataType().equals(BooleanType.INSTANCE)) { + return evalBooleanOutputExpression(input, expression); + } + // TODO: Boolean output type expressions are good enough for first preview release + // which enables partition pruning and file skipping using file stats. + + throw new UnsupportedOperationException("not yet implemented"); + } + + @Override + public void close() { /* nothing to close */ } + } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileReadContext.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileReadContext.java index 5e25fa7d53..a3ec7e97e6 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileReadContext.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileReadContext.java @@ -21,8 +21,7 @@ import io.delta.kernel.data.Row; public class DefaultFileReadContext - implements FileReadContext -{ + implements FileReadContext { private final Row scanFileRow; public DefaultFileReadContext(Row scanFileRow) { diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileSystemClient.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileSystemClient.java index fc0240db21..001e9f81d9 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileSystemClient.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultFileSystemClient.java @@ -34,8 +34,7 @@ import io.delta.kernel.utils.Utils; public class DefaultFileSystemClient - implements FileSystemClient -{ + implements FileSystemClient { private final Configuration hadoopConf; public DefaultFileSystemClient(Configuration hadoopConf) { @@ -72,6 +71,12 @@ public CloseableIterator listFrom(String filePath) { } } + @Override + public CloseableIterator readFiles( + CloseableIterator>> iter) { + return iter.map(elem -> getStream(elem._1, elem._2._1, elem._2._2)); + } + private ByteArrayInputStream getStream(String filePath, Integer offset, Integer size) { Path path = new Path(filePath); try { @@ -91,10 +96,4 @@ private ByteArrayInputStream getStream(String filePath, Integer offset, Integer "Could not resolve the FileSystem for path %s", filePath), ex); } } - - @Override - public CloseableIterator readFiles( - CloseableIterator>> iter) { - return iter.map(elem -> getStream(elem._1, elem._2._1, elem._2._2)); - } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultJsonHandler.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultJsonHandler.java index ae88806e3a..798e400006 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultJsonHandler.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultJsonHandler.java @@ -49,8 +49,7 @@ public class DefaultJsonHandler extends DefaultFileHandler - implements JsonHandler -{ + implements JsonHandler { private final ObjectMapper objectMapper; private final Configuration hadoopConf; private final int maxBatchSize; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultParquetHandler.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultParquetHandler.java index 706122f43c..d054c78101 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultParquetHandler.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultParquetHandler.java @@ -31,8 +31,7 @@ import io.delta.kernel.defaults.internal.parquet.ParquetBatchReader; -public class DefaultParquetHandler extends DefaultFileHandler implements ParquetHandler -{ +public class DefaultParquetHandler extends DefaultFileHandler implements ParquetHandler { private final Configuration hadoopConf; public DefaultParquetHandler(Configuration hadoopConf) { diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultTableClient.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultTableClient.java index 08913ff31e..143a99aeeb 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultTableClient.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/client/DefaultTableClient.java @@ -24,8 +24,7 @@ import io.delta.kernel.client.TableClient; public class DefaultTableClient - implements TableClient -{ + implements TableClient { private final Configuration hadoopConf; private DefaultTableClient(Configuration hadoopConf) { diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultKernelUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultKernelUtils.java index dc315609fd..d077174798 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultKernelUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/DefaultKernelUtils.java @@ -49,31 +49,6 @@ public static final MessageType pruneSchema( return new MessageType("fileSchema", pruneFields(fileSchema, deltaType)); } - private static List pruneFields(GroupType type, StructType deltaDataType) { - // prune fields including nested pruning like in pruneSchema - return deltaDataType.fields().stream() - .map(column -> { - Type subType = findSubFieldType(type, column); - if (subType != null) { - return prunedType(subType, column.getDataType()); - } else { - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private static Type prunedType(Type type, DataType deltaType) { - if (type instanceof GroupType && deltaType instanceof StructType) { - GroupType groupType = (GroupType) type; - StructType structType = (StructType) deltaType; - return groupType.withNewFields(pruneFields(groupType, structType)); - } else { - return type; - } - } - /** * Search for the Parquet type in {@code groupType} of subfield which is equivalent to * given {@code field}. @@ -165,4 +140,29 @@ public static int daysSinceEpoch(Date date) { LocalDate localDate = date.toLocalDate(); return (int) ChronoUnit.DAYS.between(EPOCH, localDate); } + + private static List pruneFields(GroupType type, StructType deltaDataType) { + // prune fields including nested pruning like in pruneSchema + return deltaDataType.fields().stream() + .map(column -> { + Type subType = findSubFieldType(type, column); + if (subType != null) { + return prunedType(subType, column.getDataType()); + } else { + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private static Type prunedType(Type type, DataType deltaType) { + if (type instanceof GroupType && deltaType instanceof StructType) { + GroupType groupType = (GroupType) type; + StructType structType = (StructType) deltaType; + return groupType.withNewFields(pruneFields(groupType, structType)); + } else { + return type; + } + } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultColumnarBatch.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultColumnarBatch.java index a48f335168..8875569a1b 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultColumnarBatch.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultColumnarBatch.java @@ -26,8 +26,7 @@ import io.delta.kernel.types.StructType; public class DefaultColumnarBatch - implements ColumnarBatch -{ + implements ColumnarBatch { private final int size; private final StructType schema; private final List columnVectors; @@ -55,7 +54,7 @@ public ColumnVector getColumnVector(int ordinal) { @Override public ColumnarBatch withNewColumn(int ordinal, StructField structField, - ColumnVector columnVector) { + ColumnVector columnVector) { if (ordinal < 0 || ordinal > columnVectors.size()) { throw new IllegalArgumentException("Invalid ordinal: " + ordinal); } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java index 5dbbb93955..887f70af1b 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/DefaultJsonRow.java @@ -37,8 +37,7 @@ import io.delta.kernel.types.StructField; import io.delta.kernel.types.StructType; -public class DefaultJsonRow implements Row -{ +public class DefaultJsonRow implements Row { private final Object[] parsedValues; private final StructType readSchema; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetConverters.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetConverters.java index 399e8c9691..05ae12d321 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetConverters.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ParquetConverters.java @@ -99,6 +99,21 @@ public static Converter createConverter( throw new UnsupportedOperationException(typeFromClient + " is not supported"); } + static boolean[] initNullabilityVector(int size) { + boolean[] nullability = new boolean[size]; + // Initialize all values as null. As Parquet calls this converter only for non-null + // values, make the corresponding value to false. + Arrays.fill(nullability, true); + + return nullability; + } + + static void setNullabilityToTrue(boolean[] nullability, int start, int end) { + // Initialize all values as null. As Parquet calls this converter only for non-null + // values, make the corresponding value to false. + Arrays.fill(nullability, start, end, true); + } + public interface BaseConverter { ColumnVector getDataColumnVector(int batchSize); @@ -493,19 +508,4 @@ public boolean moveToNextRow(long fileRowIndex) { return moveToNextRow(); } } - - static boolean[] initNullabilityVector(int size) { - boolean[] nullability = new boolean[size]; - // Initialize all values as null. As Parquet calls this converter only for non-null - // values, make the corresponding value to false. - Arrays.fill(nullability, true); - - return nullability; - } - - static void setNullabilityToTrue(boolean[] nullability, int start, int end) { - // Initialize all values as null. As Parquet calls this converter only for non-null - // values, make the corresponding value to false. - Arrays.fill(nullability, start, end, true); - } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RowConverter.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RowConverter.java index 01a1badbf2..db0848cbc7 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RowConverter.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/RowConverter.java @@ -108,26 +108,6 @@ public ColumnarBatch getDataAsColumnarBatch(int batchSize) { return batch; } - /** - * @return true if all members were null - */ - private boolean moveConvertersToNextRow(Optional fileRowIndex) { - long memberNullCount = Arrays.stream(converters) - .map(converter -> (ParquetConverters.BaseConverter) converter) - .map(converter -> { - if (fileRowIndex.isPresent() && - converter instanceof ParquetConverters.FileRowIndexColumnConverter) { - return ((ParquetConverters.FileRowIndexColumnConverter) converter) - .moveToNextRow(fileRowIndex.get()); - } else { - return converter.moveToNextRow(); - } - }) - .filter(result -> result) - .count(); - return memberNullCount == converters.length; - } - /** * @param fileRowIndex the file row index of the row processed */ @@ -166,13 +146,6 @@ public ColumnVector getDataColumnVector(int batchSize) { return vector; } - private ColumnVector[] collectMemberVectors(int batchSize) { - return Arrays.stream(converters) - .map(converter -> ((ParquetConverters.BaseConverter) converter).getDataColumnVector( - batchSize)) - .toArray(ColumnVector[]::new); - } - @Override public void resizeIfNeeded() { if (nullability.length == currentRowIndex) { @@ -187,4 +160,31 @@ public void resetWorkingState() { this.currentRowIndex = 0; this.nullability = ParquetConverters.initNullabilityVector(this.nullability.length); } + + /** + * @return true if all members were null + */ + private boolean moveConvertersToNextRow(Optional fileRowIndex) { + long memberNullCount = Arrays.stream(converters) + .map(converter -> (ParquetConverters.BaseConverter) converter) + .map(converter -> { + if (fileRowIndex.isPresent() && + converter instanceof ParquetConverters.FileRowIndexColumnConverter) { + return ((ParquetConverters.FileRowIndexColumnConverter) converter) + .moveToNextRow(fileRowIndex.get()); + } else { + return converter.moveToNextRow(); + } + }) + .filter(result -> result) + .count(); + return memberNullCount == converters.length; + } + + private ColumnVector[] collectMemberVectors(int batchSize) { + return Arrays.stream(converters) + .map(converter -> ((ParquetConverters.BaseConverter) converter).getDataColumnVector( + batchSize)) + .toArray(ColumnVector[]::new); + } } diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultExpressionHandler.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultExpressionHandler.java index 003fcb7463..750bda71cc 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultExpressionHandler.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultExpressionHandler.java @@ -89,7 +89,7 @@ public void evalLiterals() { testCases.add(Literal.of(new Timestamp(2342342342232L))); testCases.add(Literal.ofNull(TimestampType.INSTANCE)); - ColumnarBatch[] inputBatches = new ColumnarBatch[]{ + ColumnarBatch[] inputBatches = new ColumnarBatch[] { new DefaultColumnarBatch(0, inputSchema, data), new DefaultColumnarBatch(25, inputSchema, data), new DefaultColumnarBatch(128, inputSchema, data) @@ -151,7 +151,7 @@ public void evalBooleanExpressionSimple() { for (int size : Arrays.asList(26, 234, 567)) { StructType inputSchema = new StructType() .add("intType", IntegerType.INSTANCE); - ColumnVector[] data = new ColumnVector[]{ + ColumnVector[] data = new ColumnVector[] { intVector(size) }; @@ -182,7 +182,7 @@ public void evalBooleanExpressionComplex() { StructType inputSchema = new StructType() .add("intType", IntegerType.INSTANCE) .add("longType", LongType.INSTANCE); - ColumnVector[] data = new ColumnVector[]{ + ColumnVector[] data = new ColumnVector[] { intVector(size), longVector(size), }; diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultJsonHandler.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultJsonHandler.java index 69ad5c727a..9385d40c17 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultJsonHandler.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/client/TestDefaultJsonHandler.java @@ -59,8 +59,8 @@ public class TestDefaultJsonHandler { public void contextualizeFiles() throws Exception { try (CloseableIterator inputScanFiles = testFiles(); - CloseableIterator fileReadContexts = - JSON_HANDLER.contextualizeFileReads(testFiles(), Literal.TRUE)) { + CloseableIterator fileReadContexts = + JSON_HANDLER.contextualizeFileReads(testFiles(), Literal.TRUE)) { while (inputScanFiles.hasNext() || fileReadContexts.hasNext()) { assertEquals(inputScanFiles.hasNext(), fileReadContexts.hasNext()); Row inputScanFile = inputScanFiles.next(); diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/BaseIntegration.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/BaseIntegration.java index b5021a2794..1e6d6b80ae 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/BaseIntegration.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/BaseIntegration.java @@ -93,11 +93,11 @@ protected List readScanFiles( while (scanFilesBatchIter.hasNext()) { // Read data try (CloseableIterator data = - Scan.readData( - tableClient, - scanState, - scanFilesBatchIter.next().getRows(), - Optional.empty())) { + Scan.readData( + tableClient, + scanState, + scanFilesBatchIter.next().getRows(), + Optional.empty())) { while (data.hasNext()) { DataReadResult dataReadResult = data.next(); assertFalse(dataReadResult.getSelectionVector().isPresent()); @@ -128,7 +128,7 @@ protected StructType removeUnsupportedType(StructType schema) { } protected void compareEqualUnorderd(ColumnarBatch expDataBatch, - List actDataBatches) { + List actDataBatches) { Set expDataRowsMatched = new HashSet<>(); for (int actDataBatchIdx = 0; actDataBatchIdx < actDataBatches.size(); actDataBatchIdx++) { ColumnarBatch actDataBatch = actDataBatches.get(actDataBatchIdx); diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/DataBuilderUtils.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/DataBuilderUtils.java index 1d6028144e..54e46c4e24 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/DataBuilderUtils.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/DataBuilderUtils.java @@ -33,6 +33,14 @@ public static TestColumnBatchBuilder builder(StructType schema) { return new TestColumnBatchBuilder(schema); } + public static Row row(StructType structType, Object... values) { + return new TestRow(structType, values); + } + + public static Row row(StructType structType) { + return new TestRow(structType); + } + public static class TestColumnBatchBuilder { private StructType schema; private List rows = new ArrayList<>(); @@ -60,14 +68,6 @@ public ColumnarBatch build() { } } - public static Row row(StructType structType, Object... values) { - return new TestRow(structType, values); - } - - public static Row row(StructType structType) { - return new TestRow(structType); - } - private static class TestRow implements Row { private final StructType schema; private final Map values; diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/TestDeltaTableReads.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/TestDeltaTableReads.java index 94be922264..de0d5f798b 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/TestDeltaTableReads.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/integration/TestDeltaTableReads.java @@ -82,7 +82,7 @@ public void tablePrimitives() (float) i, (double) i, String.valueOf(i), - new byte[]{(byte) i, (byte) i} + new byte[] {(byte) i, (byte) i} ); } @@ -341,7 +341,7 @@ public void tableWithNameColumnMappingMode() (float) i, (double) i, String.valueOf(i), - new byte[]{(byte) i, (byte) i} + new byte[] {(byte) i, (byte) i} ); } diff --git a/kernel/kernel-defaults/src/test/java/io/delta/kernel/parquet/TestParquetBatchReader.java b/kernel/kernel-defaults/src/test/java/io/delta/kernel/parquet/TestParquetBatchReader.java index b6bc42df59..9f35038f26 100644 --- a/kernel/kernel-defaults/src/test/java/io/delta/kernel/parquet/TestParquetBatchReader.java +++ b/kernel/kernel-defaults/src/test/java/io/delta/kernel/parquet/TestParquetBatchReader.java @@ -143,6 +143,53 @@ public void readSubsetOfColumnsWithMissingColumnsInFile() readAndVerify(readSchema, 23 /* readBatchSize */); } + @Test + public void requestRowIndices() throws IOException { + String path = DefaultKernelTestUtils.getTestResourceFilePath("parquet-basic-row-indexes"); + File dir = new File(URI.create(path).getPath()); + List parquetFiles = Arrays.stream(Objects.requireNonNull(dir.listFiles())) + .filter(file -> file.getName().endsWith(".parquet")) + .map(File::getAbsolutePath) + .collect(Collectors.toList()); + + StructType readSchema = new StructType() + .add("id", LongType.INSTANCE) + .add(StructField.ROW_INDEX_COLUMN); + + Configuration conf = new Configuration(); + // Set the batch size small enough so there will be multiple batches + conf.setInt("delta.kernel.default.parquet.reader.batch-size", 2); + ParquetBatchReader reader = new ParquetBatchReader(conf); + + for (String filePath : parquetFiles) { + try (CloseableIterator iter = reader.read(filePath, readSchema)) { + while (iter.hasNext()) { + ColumnarBatch batch = iter.next(); + for (int i = 0; i < batch.getSize(); i++) { + long id = batch.getColumnVector(0).getLong(i); + long rowIndex = batch.getColumnVector(1).getLong(i); + assertEquals(id % 10, rowIndex); + } + } + } + } + + // File with multiple row-groups [0, 20000) where rowIndex = id + String filePath = DefaultKernelTestUtils.getTestResourceFilePath( + "parquet/row_index_multiple_row_groups.parquet"); + reader = new ParquetBatchReader(new Configuration()); + try (CloseableIterator iter = reader.read(filePath, readSchema)) { + while (iter.hasNext()) { + ColumnarBatch batch = iter.next(); + for (int i = 0; i < batch.getSize(); i++) { + long id = batch.getColumnVector(0).getLong(i); + long rowIndex = batch.getColumnVector(1).getLong(i); + assertEquals(id, rowIndex); + } + } + } + } + private static Configuration newConf(Optional batchSize) { Configuration conf = new Configuration(); if (batchSize.isPresent()) { @@ -404,51 +451,4 @@ private static Tuple2 getBatchForRowId( throw new IllegalArgumentException("row id is not found: " + rowId); } - - @Test - public void requestRowIndices() throws IOException { - String path = DefaultKernelTestUtils.getTestResourceFilePath("parquet-basic-row-indexes"); - File dir = new File(URI.create(path).getPath()); - List parquetFiles = Arrays.stream(Objects.requireNonNull(dir.listFiles())) - .filter(file -> file.getName().endsWith(".parquet")) - .map(File::getAbsolutePath) - .collect(Collectors.toList()); - - StructType readSchema = new StructType() - .add("id", LongType.INSTANCE) - .add(StructField.ROW_INDEX_COLUMN); - - Configuration conf = new Configuration(); - // Set the batch size small enough so there will be multiple batches - conf.setInt("delta.kernel.default.parquet.reader.batch-size", 2); - ParquetBatchReader reader = new ParquetBatchReader(conf); - - for (String filePath : parquetFiles) { - try (CloseableIterator iter = reader.read(filePath, readSchema)) { - while (iter.hasNext()) { - ColumnarBatch batch = iter.next(); - for (int i = 0; i < batch.getSize(); i++) { - long id = batch.getColumnVector(0).getLong(i); - long rowIndex = batch.getColumnVector(1).getLong(i); - assertEquals(id % 10, rowIndex); - } - } - } - } - - // File with multiple row-groups [0, 20000) where rowIndex = id - String filePath = DefaultKernelTestUtils.getTestResourceFilePath( - "parquet/row_index_multiple_row_groups.parquet"); - reader = new ParquetBatchReader(new Configuration()); - try (CloseableIterator iter = reader.read(filePath, readSchema)) { - while (iter.hasNext()) { - ColumnarBatch batch = iter.next(); - for (int i = 0; i < batch.getSize(); i++) { - long id = batch.getColumnVector(0).getLong(i); - long rowIndex = batch.getColumnVector(1).getLong(i); - assertEquals(id, rowIndex); - } - } - } - } } diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000029.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000029.json index b67bb6516d..9b5ebcfe50 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000029.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000029.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493467801,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#32551L = 308)\"]"},"readVersion":28,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1410","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"51e5335a-cd2c-467c-93b3-44b1731e03a6"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"l$m[rf}7BENh@zObPYMw","offset":1,"sizeInBytes":44,"cardinality":6}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493467650,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"?i@EYwa-J5RbtEIPJ.Gl","offset":1,"sizeInBytes":42,"cardinality":5}}} +{ + "commitInfo": { + "timestamp": 1687493467801, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#32551L = 308)\"]" + }, + "readVersion": 28, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1410", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "51e5335a-cd2c-467c-93b3-44b1731e03a6" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "l$m[rf}7BENh@zObPYMw", + "offset": 1, + "sizeInBytes": 44, + "cardinality": 6 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493467650, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "?i@EYwa-J5RbtEIPJ.Gl", + "offset": 1, + "sizeInBytes": 42, + "cardinality": 5 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000030.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000030.json index c88d6c36b7..4691b2086f 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000030.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000030.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493470112,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#33672L = 319)\"]"},"readVersion":29,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1399","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"852d93d5-998e-4210-871c-6f86b6fa9d9b"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":">i1<2fv@(-V::TBC{X^$","offset":1,"sizeInBytes":46,"cardinality":7}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493469924,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"l$m[rf}7BENh@zObPYMw","offset":1,"sizeInBytes":44,"cardinality":6}}} +{ + "commitInfo": { + "timestamp": 1687493470112, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#33672L = 319)\"]" + }, + "readVersion": 29, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1399", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "852d93d5-998e-4210-871c-6f86b6fa9d9b" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": ">i1<2fv@(-V::TBC{X^$", + "offset": 1, + "sizeInBytes": 46, + "cardinality": 7 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493469924, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "l$m[rf}7BENh@zObPYMw", + "offset": 1, + "sizeInBytes": 44, + "cardinality": 6 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000031.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000031.json index 6513598453..94e33a6b57 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000031.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000031.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493476087,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#34882L = 330)\"]"},"readVersion":30,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"3310","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"982792fe-4bd7-404a-a7bd-19113e7ba7da"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":")y]bjYT7n0T&3ZI7fsnR","offset":1,"sizeInBytes":48,"cardinality":8}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493475923,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":">i1<2fv@(-V::TBC{X^$","offset":1,"sizeInBytes":46,"cardinality":7}}} +{ + "commitInfo": { + "timestamp": 1687493476087, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#34882L = 330)\"]" + }, + "readVersion": 30, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "3310", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "982792fe-4bd7-404a-a7bd-19113e7ba7da" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": ")y]bjYT7n0T&3ZI7fsnR", + "offset": 1, + "sizeInBytes": 48, + "cardinality": 8 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493475923, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": ">i1<2fv@(-V::TBC{X^$", + "offset": 1, + "sizeInBytes": 46, + "cardinality": 7 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000032.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000032.json index 951294f35f..b64f049824 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000032.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000032.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493478659,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#36283L = 341)\"]"},"readVersion":31,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1489","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"19b580df-8671-46f3-bedf-f0ba70b45a64"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"B(bfMgm}tEXY:Biw>Sff","offset":1,"sizeInBytes":50,"cardinality":9}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493478493,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":")y]bjYT7n0T&3ZI7fsnR","offset":1,"sizeInBytes":48,"cardinality":8}}} +{ + "commitInfo": { + "timestamp": 1687493478659, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#36283L = 341)\"]" + }, + "readVersion": 31, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1489", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "19b580df-8671-46f3-bedf-f0ba70b45a64" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "B(bfMgm}tEXY:Biw>Sff", + "offset": 1, + "sizeInBytes": 50, + "cardinality": 9 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493478493, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": ")y]bjYT7n0T&3ZI7fsnR", + "offset": 1, + "sizeInBytes": 48, + "cardinality": 8 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000033.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000033.json index 98152737d2..0225209352 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000033.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000033.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493481125,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#37404L = 352)\"]"},"readVersion":32,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1406","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"88f23969-ecda-4300-ad0a-0522b3c50277"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"o0UC@Ov}pLUd!8XlnsU/","offset":1,"sizeInBytes":52,"cardinality":10}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493480901,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"B(bfMgm}tEXY:Biw>Sff","offset":1,"sizeInBytes":50,"cardinality":9}}} +{ + "commitInfo": { + "timestamp": 1687493481125, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#37404L = 352)\"]" + }, + "readVersion": 32, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1406", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "88f23969-ecda-4300-ad0a-0522b3c50277" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "o0UC@Ov}pLUd!8XlnsU/", + "offset": 1, + "sizeInBytes": 52, + "cardinality": 10 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493480901, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "B(bfMgm}tEXY:Biw>Sff", + "offset": 1, + "sizeInBytes": 50, + "cardinality": 9 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000034.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000034.json index 2dbac66cf6..0c10df211d 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000034.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000034.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493483736,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#38525L = 363)\"]"},"readVersion":33,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1555","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c4c917d0-94d2-4eab-9e3b-c38a00d8d30f"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"[W5]y}RT#DZ1e^jIu+&k","offset":1,"sizeInBytes":54,"cardinality":11}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493483543,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"o0UC@Ov}pLUd!8XlnsU/","offset":1,"sizeInBytes":52,"cardinality":10}}} +{ + "commitInfo": { + "timestamp": 1687493483736, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#38525L = 363)\"]" + }, + "readVersion": 33, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1555", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "c4c917d0-94d2-4eab-9e3b-c38a00d8d30f" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "[W5]y}RT#DZ1e^jIu+&k", + "offset": 1, + "sizeInBytes": 54, + "cardinality": 11 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493483543, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "o0UC@Ov}pLUd!8XlnsU/", + "offset": 1, + "sizeInBytes": 52, + "cardinality": 10 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000035.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000035.json index ad24fc4b16..40f9d07d88 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000035.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000035.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493486709,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#39646L = 374)\"]"},"readVersion":34,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"2087","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"ae9e26f7-5c42-4a11-ae83-28913ac79aaf"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"}Vi)6cLx4bSSkUi37-Sb","offset":1,"sizeInBytes":56,"cardinality":12}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493486280,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"[W5]y}RT#DZ1e^jIu+&k","offset":1,"sizeInBytes":54,"cardinality":11}}} +{ + "commitInfo": { + "timestamp": 1687493486709, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#39646L = 374)\"]" + }, + "readVersion": 34, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "2087", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "ae9e26f7-5c42-4a11-ae83-28913ac79aaf" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "}Vi)6cLx4bSSkUi37-Sb", + "offset": 1, + "sizeInBytes": 56, + "cardinality": 12 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493486280, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "[W5]y}RT#DZ1e^jIu+&k", + "offset": 1, + "sizeInBytes": 54, + "cardinality": 11 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000036.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000036.json index 873532e376..706805e96b 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000036.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000036.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493488949,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#40767L = 385)\"]"},"readVersion":35,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1387","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"f8cfc704-8c61-470b-a00b-675db4744a4d"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"QV![s>Cg)KJc8r+c2","offset":1,"sizeInBytes":58,"cardinality":13}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493488645,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"}Vi)6cLx4bSSkUi37-Sb","offset":1,"sizeInBytes":56,"cardinality":12}}} +{ + "commitInfo": { + "timestamp": 1687493488949, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#40767L = 385)\"]" + }, + "readVersion": 35, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1387", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "f8cfc704-8c61-470b-a00b-675db4744a4d" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "QV![s>Cg)KJc8r+c2", + "offset": 1, + "sizeInBytes": 58, + "cardinality": 13 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493488645, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "}Vi)6cLx4bSSkUi37-Sb", + "offset": 1, + "sizeInBytes": 56, + "cardinality": 12 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000037.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000037.json index d7f7910cd1..ec43bc4cfb 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000037.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000037.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493491174,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#41888L = 396)\"]"},"readVersion":36,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1432","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"60dd6745-b035-4fd4-9c74-cae1ad56c998"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"=pfvKB2VCg)KJc8r+c2","offset":1,"sizeInBytes":58,"cardinality":13}}} +{ + "commitInfo": { + "timestamp": 1687493491174, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#41888L = 396)\"]" + }, + "readVersion": 36, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1432", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "60dd6745-b035-4fd4-9c74-cae1ad56c998" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "=pfvKB2VCg)KJc8r+c2", + "offset": 1, + "sizeInBytes": 58, + "cardinality": 13 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000038.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000038.json index 48f105b44e..685330206c 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000038.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000038.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493493743,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#43009L = 407)\"]"},"readVersion":37,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1747","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"68f77edd-106f-4a66-a2fd-2cc53a1e8e2c"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"188rU5d7$uQnz8Cw*&JR","offset":1,"sizeInBytes":62,"cardinality":15}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493493431,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"=pfvKB2VckU","offset":1,"sizeInBytes":68,"cardinality":18}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493502653,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"^wgAJ=4X/TIzz&%qDElT","offset":1,"sizeInBytes":66,"cardinality":17}}} +{ + "commitInfo": { + "timestamp": 1687493502795, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#46461L = 440)\"]" + }, + "readVersion": 40, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1990", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "d4d539ae-e6f1-4a5c-9e49-839d5e24c54e" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "5TLR.gkXvkTLoklq>ckU", + "offset": 1, + "sizeInBytes": 68, + "cardinality": 18 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493502653, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "^wgAJ=4X/TIzz&%qDElT", + "offset": 1, + "sizeInBytes": 66, + "cardinality": 17 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000042.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000042.json index c07ebac128..0002472556 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000042.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000042.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493505010,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#47862L = 451)\"]"},"readVersion":41,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1328","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"c9eadb2b-ac9c-4b31-9b38-923978074394"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"1ENM$^ul]}VH<&ub7J5y","offset":1,"sizeInBytes":70,"cardinality":19}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493504829,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"5TLR.gkXvkTLoklq>ckU","offset":1,"sizeInBytes":68,"cardinality":18}}} +{ + "commitInfo": { + "timestamp": 1687493505010, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#47862L = 451)\"]" + }, + "readVersion": 41, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1328", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "c9eadb2b-ac9c-4b31-9b38-923978074394" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "1ENM$^ul]}VH<&ub7J5y", + "offset": 1, + "sizeInBytes": 70, + "cardinality": 19 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493504829, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "5TLR.gkXvkTLoklq>ckU", + "offset": 1, + "sizeInBytes": 68, + "cardinality": 18 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000043.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000043.json index 88c3900c88..56e08fa6a4 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000043.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000043.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493507835,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#48983L = 462)\"]"},"readVersion":42,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1977","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"81b2bee3-440a-4051-92ca-c7c9df8fcd02"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"1lI0]tO]*HV].E0P<8T2","offset":1,"sizeInBytes":72,"cardinality":20}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493507663,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"1ENM$^ul]}VH<&ub7J5y","offset":1,"sizeInBytes":70,"cardinality":19}}} +{ + "commitInfo": { + "timestamp": 1687493507835, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#48983L = 462)\"]" + }, + "readVersion": 42, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1977", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "81b2bee3-440a-4051-92ca-c7c9df8fcd02" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "1lI0]tO]*HV].E0P<8T2", + "offset": 1, + "sizeInBytes": 72, + "cardinality": 20 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493507663, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "1ENM$^ul]}VH<&ub7J5y", + "offset": 1, + "sizeInBytes": 70, + "cardinality": 19 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000044.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000044.json index 15af43ab01..22af197e20 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000044.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000044.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493510227,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#50104L = 473)\"]"},"readVersion":43,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1453","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5096139c-7d93-40aa-a573-37088d859c0c"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"8RfaCZ!EkOFD(^Syd?cC","offset":1,"sizeInBytes":74,"cardinality":21}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493510052,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"1lI0]tO]*HV].E0P<8T2","offset":1,"sizeInBytes":72,"cardinality":20}}} +{ + "commitInfo": { + "timestamp": 1687493510227, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#50104L = 473)\"]" + }, + "readVersion": 43, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1453", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "5096139c-7d93-40aa-a573-37088d859c0c" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "8RfaCZ!EkOFD(^Syd?cC", + "offset": 1, + "sizeInBytes": 74, + "cardinality": 21 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493510052, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "1lI0]tO]*HV].E0P<8T2", + "offset": 1, + "sizeInBytes": 72, + "cardinality": 20 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000045.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000045.json index 22aa22fb13..bc1ba82b4d 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000045.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000045.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493513120,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#51225L = 484)\"]"},"readVersion":44,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1945","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"5efa6fb6-29e3-46c9-8a4a-2d82df02da68"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"Yq}%xtpq&YTHdjxg?Osr","offset":1,"sizeInBytes":76,"cardinality":22}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493512712,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"8RfaCZ!EkOFD(^Syd?cC","offset":1,"sizeInBytes":74,"cardinality":21}}} +{ + "commitInfo": { + "timestamp": 1687493513120, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#51225L = 484)\"]" + }, + "readVersion": 44, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1945", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "5efa6fb6-29e3-46c9-8a4a-2d82df02da68" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "Yq}%xtpq&YTHdjxg?Osr", + "offset": 1, + "sizeInBytes": 76, + "cardinality": 22 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493512712, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "8RfaCZ!EkOFD(^Syd?cC", + "offset": 1, + "sizeInBytes": 74, + "cardinality": 21 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000046.json b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000046.json index 3a4eb29ab2..6fdce0897e 100644 --- a/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000046.json +++ b/kernel/kernel-defaults/src/test/resources/basic-dv-with-checkpoint/_delta_log/00000000000000000046.json @@ -1,3 +1,60 @@ -{"commitInfo":{"timestamp":1687493515976,"operation":"DELETE","operationParameters":{"predicate":"[\"(id#52346L = 495)\"]"},"readVersion":45,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numRemovedFiles":"0","numRemovedBytes":"0","numCopiedRows":"0","numAddedChangeFiles":"0","executionTimeMs":"1419","numDeletedRows":"1","scanTimeMs":"0","numAddedFiles":"0","numAddedBytes":"0","rewriteTimeMs":"0"},"engineInfo":"Apache-Spark/3.4.0 Delta-Lake/2.4.0","txnId":"deeed50a-1264-4c7e-9e26-b3217e7779dd"}} -{"add":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","partitionValues":{},"size":1495,"modificationTime":1687493351000,"dataChange":true,"stats":"{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}","deletionVector":{"storageType":"u","pathOrInlineDv":"+Ry*Wio9zqLW9MIvM6Su","offset":1,"sizeInBytes":78,"cardinality":23}}} -{"remove":{"path":"part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet","deletionTimestamp":1687493515806,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{},"size":1495,"deletionVector":{"storageType":"u","pathOrInlineDv":"Yq}%xtpq&YTHdjxg?Osr","offset":1,"sizeInBytes":76,"cardinality":22}}} +{ + "commitInfo": { + "timestamp": 1687493515976, + "operation": "DELETE", + "operationParameters": { + "predicate": "[\"(id#52346L = 495)\"]" + }, + "readVersion": 45, + "isolationLevel": "Serializable", + "isBlindAppend": false, + "operationMetrics": { + "numRemovedFiles": "0", + "numRemovedBytes": "0", + "numCopiedRows": "0", + "numAddedChangeFiles": "0", + "executionTimeMs": "1419", + "numDeletedRows": "1", + "scanTimeMs": "0", + "numAddedFiles": "0", + "numAddedBytes": "0", + "rewriteTimeMs": "0" + }, + "engineInfo": "Apache-Spark/3.4.0 Delta-Lake/2.4.0", + "txnId": "deeed50a-1264-4c7e-9e26-b3217e7779dd" + } +} +{ + "add": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "partitionValues": {}, + "size": 1495, + "modificationTime": 1687493351000, + "dataChange": true, + "stats": "{\"numRecords\":250,\"minValues\":{\"id\":250},\"maxValues\":{\"id\":499},\"nullCount\":{\"id\":0},\"tightBounds\":false}", + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "+Ry*Wio9zqLW9MIvM6Su", + "offset": 1, + "sizeInBytes": 78, + "cardinality": 23 + } + } +} +{ + "remove": { + "path": "part-00001-c94c50bd-c7bb-4c0d-b6cb-958707d77d01-c000.snappy.parquet", + "deletionTimestamp": 1687493515806, + "dataChange": true, + "extendedFileMetadata": true, + "partitionValues": {}, + "size": 1495, + "deletionVector": { + "storageType": "u", + "pathOrInlineDv": "Yq}%xtpq&YTHdjxg?Osr", + "offset": 1, + "sizeInBytes": 76, + "cardinality": 22 + } + } +} diff --git a/kernel/kernel-defaults/src/test/resources/data-reader-partition-values-column-mapping-name/_delta_log/00000000000000000000.json b/kernel/kernel-defaults/src/test/resources/data-reader-partition-values-column-mapping-name/_delta_log/00000000000000000000.json index 0d21975197..52bcd61f8e 100644 --- a/kernel/kernel-defaults/src/test/resources/data-reader-partition-values-column-mapping-name/_delta_log/00000000000000000000.json +++ b/kernel/kernel-defaults/src/test/resources/data-reader-partition-values-column-mapping-name/_delta_log/00000000000000000000.json @@ -1,6 +1,119 @@ -{"commitInfo":{"timestamp":1687761154342,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"as_int\",\"as_long\",\"as_byte\",\"as_short\",\"as_boolean\",\"as_float\",\"as_double\",\"as_string\",\"as_date\",\"as_timestamp\",\"as_big_decimal\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"3","numOutputRows":"3","numOutputBytes":"1896"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.3.0","txnId":"8ddbc378-38bd-4992-b394-c73162a776ec"}} -{"protocol":{"minReaderVersion":2,"minWriterVersion":5}} -{"metaData":{"id":"85380a11-828f-4831-b58f-219ffc825181","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-29f826c0-7fff-4e5f-bc11-44a6975c7708\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-7781d665-6951-4244-b9bc-a28e477e2d57\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-3463c48b-4b94-4500-b14f-4a554284b94f\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-05f332c4-ebdb-4437-9e80-e23f92bee4a2\"}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-c025b8f8-481c-4db2-8932-f37129146ceb\"}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-bd963d5f-2199-4700-b5d6-0759bd7a9d90\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.physicalName\":\"col-01ec4063-ed54-41db-805e-ebfd9b9a6e67\"}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.physicalName\":\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\"}}]}","partitionColumns":["as_int","as_long","as_byte","as_short","as_boolean","as_float","as_double","as_string","as_date","as_timestamp","as_big_decimal"],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"12"},"createdTime":1687761153419}} -{"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=1/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=1/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=1/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=1/col-7781d665-6951-4244-b9bc-a28e477e2d57=false/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=1.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=1.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=1/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=1/part-00000-c9d9ab23-0f5c-4a12-837f-b709c5037905.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":"1","col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":"1","col-29f826c0-7fff-4e5f-bc11-44a6975c7708":"1","col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":"1","col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":"1","col-3463c48b-4b94-4500-b14f-4a554284b94f":"1.0","col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":"1.0","col-7781d665-6951-4244-b9bc-a28e477e2d57":"false","col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":"1","col-c025b8f8-481c-4db2-8932-f37129146ceb":"2021-09-08","col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":"2021-09-08 11:11:11"},"size":632,"modificationTime":1687761154332,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} -{"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=__HIVE_DEFAULT_PARTITION__/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=__HIVE_DEFAULT_PARTITION__/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=__HIVE_DEFAULT_PARTITION__/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=__HIVE_DEFAULT_PARTITION__/col-7781d665-6951-4244-b9bc-a28e477e2d57=__HIVE_DEFAULT_PARTITION__/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=__HIVE_DEFAULT_PARTITION__/col-3463c48b-4b94-4500-b14f-4a554284b94f=__HIVE_DEFAULT_PARTITION__/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=__HIVE_DEFAULT_PARTITION__/col-c025b8f8-481c-4db2-8932-f37129146ceb=__HIVE_DEFAULT_PARTITION__/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=__HIVE_DEFAULT_PARTITION__/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=__HIVE_DEFAULT_PARTITION__/part-00001-dac9e981-94cc-4dc5-8d01-2cbae8ef69c6.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":null,"col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":null,"col-29f826c0-7fff-4e5f-bc11-44a6975c7708":null,"col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":null,"col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":null,"col-3463c48b-4b94-4500-b14f-4a554284b94f":null,"col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":null,"col-7781d665-6951-4244-b9bc-a28e477e2d57":null,"col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":null,"col-c025b8f8-481c-4db2-8932-f37129146ceb":null,"col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":null},"size":632,"modificationTime":1687761154335,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} -{"add":{"path":"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=0/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=0/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=0/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=0/col-7781d665-6951-4244-b9bc-a28e477e2d57=true/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=0.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=0.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=0/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=0/part-00002-e0842c02-93d2-4c38-b041-fc88b581688b.c000.snappy.parquet","partitionValues":{"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed":"0","col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa":"0","col-29f826c0-7fff-4e5f-bc11-44a6975c7708":"0","col-01ec4063-ed54-41db-805e-ebfd9b9a6e67":"0","col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08":"0","col-3463c48b-4b94-4500-b14f-4a554284b94f":"0.0","col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae":"0.0","col-7781d665-6951-4244-b9bc-a28e477e2d57":"true","col-05f332c4-ebdb-4437-9e80-e23f92bee4a2":"0","col-c025b8f8-481c-4db2-8932-f37129146ceb":"2021-09-08","col-bd963d5f-2199-4700-b5d6-0759bd7a9d90":"2021-09-08 11:11:11"},"size":632,"modificationTime":1687761154336,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}"}} +{ + "commitInfo": { + "timestamp": 1687761154342, + "operation": "WRITE", + "operationParameters": { + "mode": "ErrorIfExists", + "partitionBy": "[\"as_int\",\"as_long\",\"as_byte\",\"as_short\",\"as_boolean\",\"as_float\",\"as_double\",\"as_string\",\"as_date\",\"as_timestamp\",\"as_big_decimal\"]" + }, + "isolationLevel": "Serializable", + "isBlindAppend": true, + "operationMetrics": { + "numFiles": "3", + "numOutputRows": "3", + "numOutputBytes": "1896" + }, + "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.3.0", + "txnId": "8ddbc378-38bd-4992-b394-c73162a776ec" + } +} +{ + "protocol": { + "minReaderVersion": 2, + "minWriterVersion": 5 + } +} +{ + "metaData": { + "id": "85380a11-828f-4831-b58f-219ffc825181", + "format": { + "provider": "parquet", + "options": {} + }, + "schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-25948c99-9f51-4e05-9f9e-b4f7042f75ed\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-29f826c0-7fff-4e5f-bc11-44a6975c7708\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-7781d665-6951-4244-b9bc-a28e477e2d57\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-3463c48b-4b94-4500-b14f-4a554284b94f\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-05f332c4-ebdb-4437-9e80-e23f92bee4a2\"}},{\"name\":\"as_date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-c025b8f8-481c-4db2-8932-f37129146ceb\"}},{\"name\":\"as_timestamp\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-bd963d5f-2199-4700-b5d6-0759bd7a9d90\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":11,\"delta.columnMapping.physicalName\":\"col-01ec4063-ed54-41db-805e-ebfd9b9a6e67\"}},{\"name\":\"value\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":12,\"delta.columnMapping.physicalName\":\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\"}}]}", + "partitionColumns": [ + "as_int", + "as_long", + "as_byte", + "as_short", + "as_boolean", + "as_float", + "as_double", + "as_string", + "as_date", + "as_timestamp", + "as_big_decimal" + ], + "configuration": { + "delta.columnMapping.mode": "name", + "delta.columnMapping.maxColumnId": "12" + }, + "createdTime": 1687761153419 + } +} +{ + "add": { + "path": "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=1/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=1/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=1/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=1/col-7781d665-6951-4244-b9bc-a28e477e2d57=false/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=1.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=1.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=1/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=1/part-00000-c9d9ab23-0f5c-4a12-837f-b709c5037905.c000.snappy.parquet", + "partitionValues": { + "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed": "1", + "col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa": "1", + "col-29f826c0-7fff-4e5f-bc11-44a6975c7708": "1", + "col-01ec4063-ed54-41db-805e-ebfd9b9a6e67": "1", + "col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08": "1", + "col-3463c48b-4b94-4500-b14f-4a554284b94f": "1.0", + "col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae": "1.0", + "col-7781d665-6951-4244-b9bc-a28e477e2d57": "false", + "col-05f332c4-ebdb-4437-9e80-e23f92bee4a2": "1", + "col-c025b8f8-481c-4db2-8932-f37129146ceb": "2021-09-08", + "col-bd963d5f-2199-4700-b5d6-0759bd7a9d90": "2021-09-08 11:11:11" + }, + "size": 632, + "modificationTime": 1687761154332, + "dataChange": true, + "stats": "{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"1\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}" + } +} +{ + "add": { + "path": "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=__HIVE_DEFAULT_PARTITION__/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=__HIVE_DEFAULT_PARTITION__/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=__HIVE_DEFAULT_PARTITION__/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=__HIVE_DEFAULT_PARTITION__/col-7781d665-6951-4244-b9bc-a28e477e2d57=__HIVE_DEFAULT_PARTITION__/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=__HIVE_DEFAULT_PARTITION__/col-3463c48b-4b94-4500-b14f-4a554284b94f=__HIVE_DEFAULT_PARTITION__/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=__HIVE_DEFAULT_PARTITION__/col-c025b8f8-481c-4db2-8932-f37129146ceb=__HIVE_DEFAULT_PARTITION__/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=__HIVE_DEFAULT_PARTITION__/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=__HIVE_DEFAULT_PARTITION__/part-00001-dac9e981-94cc-4dc5-8d01-2cbae8ef69c6.c000.snappy.parquet", + "partitionValues": { + "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed": null, + "col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa": null, + "col-29f826c0-7fff-4e5f-bc11-44a6975c7708": null, + "col-01ec4063-ed54-41db-805e-ebfd9b9a6e67": null, + "col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08": null, + "col-3463c48b-4b94-4500-b14f-4a554284b94f": null, + "col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae": null, + "col-7781d665-6951-4244-b9bc-a28e477e2d57": null, + "col-05f332c4-ebdb-4437-9e80-e23f92bee4a2": null, + "col-c025b8f8-481c-4db2-8932-f37129146ceb": null, + "col-bd963d5f-2199-4700-b5d6-0759bd7a9d90": null + }, + "size": 632, + "modificationTime": 1687761154335, + "dataChange": true, + "stats": "{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"2\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}" + } +} +{ + "add": { + "path": "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed=0/col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa=0/col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08=0/col-29f826c0-7fff-4e5f-bc11-44a6975c7708=0/col-7781d665-6951-4244-b9bc-a28e477e2d57=true/col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae=0.0/col-3463c48b-4b94-4500-b14f-4a554284b94f=0.0/col-05f332c4-ebdb-4437-9e80-e23f92bee4a2=0/col-c025b8f8-481c-4db2-8932-f37129146ceb=2021-09-08/col-bd963d5f-2199-4700-b5d6-0759bd7a9d90=2021-09-08%2011%253A11%253A11/col-01ec4063-ed54-41db-805e-ebfd9b9a6e67=0/part-00002-e0842c02-93d2-4c38-b041-fc88b581688b.c000.snappy.parquet", + "partitionValues": { + "col-25948c99-9f51-4e05-9f9e-b4f7042f75ed": "0", + "col-6e87b90d-f5df-4dcc-91a1-7a43fa3173fa": "0", + "col-29f826c0-7fff-4e5f-bc11-44a6975c7708": "0", + "col-01ec4063-ed54-41db-805e-ebfd9b9a6e67": "0", + "col-e3e3dce4-fbd4-4a52-a0bd-5d54af7a7a08": "0", + "col-3463c48b-4b94-4500-b14f-4a554284b94f": "0.0", + "col-b72a5284-7c06-47f9-9e37-c88c3b54c6ae": "0.0", + "col-7781d665-6951-4244-b9bc-a28e477e2d57": "true", + "col-05f332c4-ebdb-4437-9e80-e23f92bee4a2": "0", + "col-c025b8f8-481c-4db2-8932-f37129146ceb": "2021-09-08", + "col-bd963d5f-2199-4700-b5d6-0759bd7a9d90": "2021-09-08 11:11:11" + }, + "size": 632, + "modificationTime": 1687761154336, + "dataChange": true, + "stats": "{\"numRecords\":1,\"minValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"maxValues\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":\"0\"},\"nullCount\":{\"col-9045d285-9170-4b1b-acc4-1ce4ee176916\":0}}" + } +} diff --git a/kernel/kernel-defaults/src/test/resources/data-reader-primitives-column-mapping-name/_delta_log/00000000000000000000.json b/kernel/kernel-defaults/src/test/resources/data-reader-primitives-column-mapping-name/_delta_log/00000000000000000000.json index a2cfc7de38..2b69f095c4 100644 --- a/kernel/kernel-defaults/src/test/resources/data-reader-primitives-column-mapping-name/_delta_log/00000000000000000000.json +++ b/kernel/kernel-defaults/src/test/resources/data-reader-primitives-column-mapping-name/_delta_log/00000000000000000000.json @@ -1,5 +1,61 @@ -{"commitInfo":{"timestamp":1687757789720,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"2","numOutputRows":"11","numOutputBytes":"8996"},"engineInfo":"Apache-Spark/3.3.1 Delta-Lake/2.3.0","txnId":"95e57353-d8fc-4e4e-a7ee-0c56559054d9"}} -{"protocol":{"minReaderVersion":2,"minWriterVersion":5}} -{"metaData":{"id":"02a552b7-5f9f-4fef-a992-ddc436e735cf","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\"}},{\"name\":\"as_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-7441db29-eefe-4fed-b11c-a0886325267e\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\"}}]}","partitionColumns":[],"configuration":{"delta.columnMapping.mode":"name","delta.columnMapping.maxColumnId":"10"},"createdTime":1687757788998}} -{"add":{"path":"part-00000-dedd3195-6cd1-451d-83b8-fe0028f9b2b6-c000.snappy.parquet","partitionValues":{},"size":4542,"modificationTime":1687757789686,"dataChange":true,"stats":"{\"numRecords\":6,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":4,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":4,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":4,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":4,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":4.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":4.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"4\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":4},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":9,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":9,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":9,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":9,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":9.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":9.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"9\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":9},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":0,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":0,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0}}"}} -{"add":{"path":"part-00001-d8bdfc55-29fe-40bc-bfe4-f7732d559aa9-c000.snappy.parquet","partitionValues":{},"size":4454,"modificationTime":1687757789686,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"0\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":3,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":3,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":3,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":3,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":3.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":3.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"3\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":3},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":1,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":1,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":1,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":1,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":1,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":1,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":1,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":1,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":1,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":1}}"}} +{ + "commitInfo": { + "timestamp": 1687757789720, + "operation": "WRITE", + "operationParameters": { + "mode": "ErrorIfExists", + "partitionBy": "[]" + }, + "isolationLevel": "Serializable", + "isBlindAppend": true, + "operationMetrics": { + "numFiles": "2", + "numOutputRows": "11", + "numOutputBytes": "8996" + }, + "engineInfo": "Apache-Spark/3.3.1 Delta-Lake/2.3.0", + "txnId": "95e57353-d8fc-4e4e-a7ee-0c56559054d9" + } +} +{ + "protocol": { + "minReaderVersion": 2, + "minWriterVersion": 5 + } +} +{ + "metaData": { + "id": "02a552b7-5f9f-4fef-a992-ddc436e735cf", + "format": { + "provider": "parquet", + "options": {} + }, + "schemaString": "{\"type\":\"struct\",\"fields\":[{\"name\":\"as_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\"}},{\"name\":\"as_long\",\"type\":\"long\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\"}},{\"name\":\"as_byte\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\"}},{\"name\":\"as_short\",\"type\":\"short\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":4,\"delta.columnMapping.physicalName\":\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\"}},{\"name\":\"as_boolean\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":5,\"delta.columnMapping.physicalName\":\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\"}},{\"name\":\"as_float\",\"type\":\"float\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":6,\"delta.columnMapping.physicalName\":\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\"}},{\"name\":\"as_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":7,\"delta.columnMapping.physicalName\":\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\"}},{\"name\":\"as_string\",\"type\":\"string\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":8,\"delta.columnMapping.physicalName\":\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\"}},{\"name\":\"as_binary\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":9,\"delta.columnMapping.physicalName\":\"col-7441db29-eefe-4fed-b11c-a0886325267e\"}},{\"name\":\"as_big_decimal\",\"type\":\"decimal(1,0)\",\"nullable\":true,\"metadata\":{\"delta.columnMapping.id\":10,\"delta.columnMapping.physicalName\":\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\"}}]}", + "partitionColumns": [], + "configuration": { + "delta.columnMapping.mode": "name", + "delta.columnMapping.maxColumnId": "10" + }, + "createdTime": 1687757788998 + } +} +{ + "add": { + "path": "part-00000-dedd3195-6cd1-451d-83b8-fe0028f9b2b6-c000.snappy.parquet", + "partitionValues": {}, + "size": 4542, + "modificationTime": 1687757789686, + "dataChange": true, + "stats": "{\"numRecords\":6,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":4,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":4,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":4,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":4,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":4.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":4.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"4\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":4},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":9,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":9,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":9,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":9,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":9.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":9.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"9\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":9},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":0,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":0,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0}}" + } +} +{ + "add": { + "path": "part-00001-d8bdfc55-29fe-40bc-bfe4-f7732d559aa9-c000.snappy.parquet", + "partitionValues": {}, + "size": 4454, + "modificationTime": 1687757789686, + "dataChange": true, + "stats": "{\"numRecords\":5,\"minValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":0,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":0,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":0,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":0,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":0.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":0.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"0\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":0},\"maxValues\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":3,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":3,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":3,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":3,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":3.0,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":3.0,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":\"3\",\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":3},\"nullCount\":{\"col-41726333-d452-47f3-b9ed-0adff6ffeabf\":1,\"col-668f7731-ddc9-4523-83cd-9c33d739c7d4\":1,\"col-6fbaa34c-ff0e-45c3-ab66-efdf765dd606\":1,\"col-bbfb738d-7298-4c00-a465-44b80ca3c97a\":1,\"col-eded3bff-704e-4046-97e6-1395b1e38f2a\":1,\"col-97d3b9ba-c7e6-4f97-8e64-b4ffd3c836ca\":1,\"col-438778fc-0408-4a26-9aca-9ce1d7d3b63d\":1,\"col-ce49aefa-a5a0-4f8f-80c1-ae3ae9918ffa\":1,\"col-7441db29-eefe-4fed-b11c-a0886325267e\":1,\"col-1dbc7e88-d12e-4e0c-b35d-1d504f71207a\":1}}" + } +}