Skip to content

Commit 9839efa

Browse files
ZouxxyyZouxxyy
authored
[core] Introduce SplitGroup for SplitGenerator to optimize more rawFiles (#3059)
1 parent 2401f7e commit 9839efa

File tree

8 files changed

+158
-47
lines changed

8 files changed

+158
-47
lines changed

‎paimon-core/src/main/java/org/apache/paimon/table/source/AppendOnlySplitGenerator.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.Collections;
2727
import java.util.List;
2828
import java.util.function.Function;
29+
import java.util.stream.Collectors;
2930

3031
import static org.apache.paimon.append.AppendOnlyCompactManager.fileComparator;
3132

@@ -44,21 +45,23 @@ public AppendOnlySplitGenerator(
4445
}
4546

4647
@Override
47-
public List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> input) {
48+
public List<SplitGroup> splitForBatch(List<DataFileMeta> input) {
4849
List<DataFileMeta> files = new ArrayList<>(input);
4950
files.sort(fileComparator(bucketMode == BucketMode.UNAWARE));
5051
Function<DataFileMeta, Long> weightFunc = file -> Math.max(file.fileSize(), openFileCost);
51-
return BinPacking.packForOrdered(files, weightFunc, targetSplitSize);
52+
return BinPacking.packForOrdered(files, weightFunc, targetSplitSize).stream()
53+
.map(SplitGroup::rawConvertibleGroup)
54+
.collect(Collectors.toList());
5255
}
5356

5457
@Override
55-
public List<List<DataFileMeta>> splitForStreaming(List<DataFileMeta> files) {
58+
public List<SplitGroup> splitForStreaming(List<DataFileMeta> files) {
5659
// When the bucket mode is unaware, we spit the files as batch, because unaware-bucket table
5760
// only contains one bucket (bucket 0).
5861
if (bucketMode == BucketMode.UNAWARE) {
5962
return splitForBatch(files);
6063
} else {
61-
return Collections.singletonList(files);
64+
return Collections.singletonList(SplitGroup.rawConvertibleGroup(files));
6265
}
6366
}
6467
}

‎paimon-core/src/main/java/org/apache/paimon/table/source/MergeTreeSplitGenerator.java

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,18 @@ public MergeTreeSplitGenerator(
6161
}
6262

6363
@Override
64-
public List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> files) {
65-
if (deletionVectorsEnabled || mergeEngine == FIRST_ROW) {
64+
public List<SplitGroup> splitForBatch(List<DataFileMeta> files) {
65+
boolean rawConvertible =
66+
files.stream().allMatch(file -> file.level() != 0 && withoutDeleteRow(file));
67+
boolean oneLevel =
68+
files.stream().map(DataFileMeta::level).collect(Collectors.toSet()).size() == 1;
69+
70+
if (rawConvertible && (deletionVectorsEnabled || mergeEngine == FIRST_ROW || oneLevel)) {
6671
Function<DataFileMeta, Long> weightFunc =
6772
file -> Math.max(file.fileSize(), openFileCost);
68-
return BinPacking.packForOrdered(files, weightFunc, targetSplitSize);
73+
return BinPacking.packForOrdered(files, weightFunc, targetSplitSize).stream()
74+
.map(SplitGroup::rawConvertibleGroup)
75+
.collect(Collectors.toList());
6976
}
7077

7178
/*
@@ -93,13 +100,19 @@ public List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> files) {
93100
new IntervalPartition(files, keyComparator)
94101
.partition().stream().map(this::flatRun).collect(Collectors.toList());
95102

96-
return packSplits(sections);
103+
return packSplits(sections).stream()
104+
.map(
105+
f ->
106+
f.size() == 1 && withoutDeleteRow(f.get(0))
107+
? SplitGroup.rawConvertibleGroup(f)
108+
: SplitGroup.nonRawConvertibleGroup(f))
109+
.collect(Collectors.toList());
97110
}
98111

99112
@Override
100-
public List<List<DataFileMeta>> splitForStreaming(List<DataFileMeta> files) {
113+
public List<SplitGroup> splitForStreaming(List<DataFileMeta> files) {
101114
// We don't split streaming scan files
102-
return Collections.singletonList(files);
115+
return Collections.singletonList(SplitGroup.rawConvertibleGroup(files));
103116
}
104117

105118
private List<List<DataFileMeta>> packSplits(List<List<DataFileMeta>> sections) {
@@ -129,4 +142,8 @@ private List<DataFileMeta> flatFiles(List<List<DataFileMeta>> section) {
129142
section.forEach(files::addAll);
130143
return files;
131144
}
145+
146+
private boolean withoutDeleteRow(DataFileMeta dataFileMeta) {
147+
return dataFileMeta.deleteRowCount().map(count -> count == 0L).orElse(false);
148+
}
132149
}

‎paimon-core/src/main/java/org/apache/paimon/table/source/SplitGenerator.java

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,27 @@
2525
/** Generate splits from {@link DataFileMeta}s. */
2626
public interface SplitGenerator {
2727

28-
List<List<DataFileMeta>> splitForBatch(List<DataFileMeta> files);
28+
List<SplitGroup> splitForBatch(List<DataFileMeta> files);
2929

30-
List<List<DataFileMeta>> splitForStreaming(List<DataFileMeta> files);
30+
List<SplitGroup> splitForStreaming(List<DataFileMeta> files);
31+
32+
/** Split group. */
33+
class SplitGroup {
34+
35+
public final List<DataFileMeta> files;
36+
public final boolean rawConvertible;
37+
38+
private SplitGroup(List<DataFileMeta> files, boolean rawConvertible) {
39+
this.files = files;
40+
this.rawConvertible = rawConvertible;
41+
}
42+
43+
public static SplitGroup rawConvertibleGroup(List<DataFileMeta> files) {
44+
return new SplitGroup(files, true);
45+
}
46+
47+
public static SplitGroup nonRawConvertibleGroup(List<DataFileMeta> files) {
48+
return new SplitGroup(files, false);
49+
}
50+
}
3151
}

‎paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/IncrementalStartingScanner.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.paimon.table.source.DataSplit;
2626
import org.apache.paimon.table.source.PlanImpl;
2727
import org.apache.paimon.table.source.ScanMode;
28+
import org.apache.paimon.table.source.SplitGenerator;
2829
import org.apache.paimon.utils.Pair;
2930
import org.apache.paimon.utils.SnapshotManager;
3031

@@ -65,15 +66,15 @@ public Result scan(SnapshotReader reader) {
6566
for (Map.Entry<Pair<BinaryRow, Integer>, List<DataFileMeta>> entry : grouped.entrySet()) {
6667
BinaryRow partition = entry.getKey().getLeft();
6768
int bucket = entry.getKey().getRight();
68-
for (List<DataFileMeta> files :
69+
for (SplitGenerator.SplitGroup splitGroup :
6970
reader.splitGenerator().splitForBatch(entry.getValue())) {
7071
// TODO pass deletion files
7172
result.add(
7273
DataSplit.builder()
7374
.withSnapshot(endingSnapshotId)
7475
.withPartition(partition)
7576
.withBucket(bucket)
76-
.withDataFiles(files)
77+
.withDataFiles(splitGroup.files)
7778
.build());
7879
}
7980
}

‎paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/SnapshotReaderImpl.java

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
import java.util.function.BiConsumer;
6565
import java.util.stream.Collectors;
6666

67-
import static org.apache.paimon.CoreOptions.MergeEngine.FIRST_ROW;
6867
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
6968
import static org.apache.paimon.operation.FileStoreScan.Plan.groupByPartFiles;
7069
import static org.apache.paimon.predicate.PredicateBuilder.transformFieldMapping;
@@ -278,18 +277,24 @@ private List<DataSplit> generateSplits(
278277
.withPartition(partition)
279278
.withBucket(bucket)
280279
.isStreaming(isStreaming);
281-
List<List<DataFileMeta>> splitGroups =
280+
List<SplitGenerator.SplitGroup> splitGroups =
282281
isStreaming
283282
? splitGenerator.splitForStreaming(bucketFiles)
284283
: splitGenerator.splitForBatch(bucketFiles);
285-
for (List<DataFileMeta> dataFiles : splitGroups) {
286-
builder.withDataFiles(dataFiles)
287-
.rawFiles(convertToRawFiles(partition, bucket, dataFiles));
288-
if (deletionVectors) {
289-
IndexFileMeta deletionIndexFile =
290-
indexFileHandler
284+
285+
IndexFileMeta deletionIndexFile =
286+
deletionVectors
287+
? indexFileHandler
291288
.scan(snapshotId, DELETION_VECTORS_INDEX, partition, bucket)
292-
.orElse(null);
289+
.orElse(null)
290+
: null;
291+
for (SplitGenerator.SplitGroup splitGroup : splitGroups) {
292+
List<DataFileMeta> dataFiles = splitGroup.files;
293+
builder.withDataFiles(dataFiles);
294+
if (splitGroup.rawConvertible) {
295+
builder.rawFiles(convertToRawFiles(partition, bucket, dataFiles));
296+
}
297+
if (deletionVectors) {
293298
builder.withDataDeletionFiles(
294299
getDeletionFiles(dataFiles, deletionIndexFile));
295300
}
@@ -370,8 +375,7 @@ private Plan toChangesPlan(
370375
.withBucket(bucket)
371376
.withBeforeFiles(before)
372377
.withDataFiles(data)
373-
.isStreaming(isStreaming)
374-
.rawFiles(convertToRawFiles(part, bucket, data));
378+
.isStreaming(isStreaming);
375379
if (deletionVectors) {
376380
IndexFileMeta beforeDeletionIndex =
377381
indexFileHandler
@@ -437,21 +441,6 @@ private List<DeletionFile> getDeletionFiles(
437441
private List<RawFile> convertToRawFiles(
438442
BinaryRow partition, int bucket, List<DataFileMeta> dataFiles) {
439443
String bucketPath = pathFactory.bucketPath(partition, bucket).toString();
440-
441-
// append only or deletionVectors files can be returned
442-
if (tableSchema.primaryKeys().isEmpty() || deletionVectors || mergeEngine == FIRST_ROW) {
443-
return makeRawTableFiles(bucketPath, dataFiles);
444-
}
445-
446-
int maxLevel = options.numLevels() - 1;
447-
if (dataFiles.stream().map(DataFileMeta::level).allMatch(l -> l == maxLevel)) {
448-
return makeRawTableFiles(bucketPath, dataFiles);
449-
}
450-
451-
return Collections.emptyList();
452-
}
453-
454-
private List<RawFile> makeRawTableFiles(String bucketPath, List<DataFileMeta> dataFiles) {
455444
return dataFiles.stream()
456445
.map(f -> makeRawTableFile(bucketPath, f))
457446
.collect(Collectors.toList());

‎paimon-core/src/test/java/org/apache/paimon/io/DataFileTestUtils.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,10 +72,15 @@ public static DataFileMeta newFile() {
7272

7373
public static DataFileMeta newFile(
7474
String name, int level, int minKey, int maxKey, long maxSequence) {
75+
return newFile(name, level, minKey, maxKey, maxSequence, 0L);
76+
}
77+
78+
public static DataFileMeta newFile(
79+
String name, int level, int minKey, int maxKey, long maxSequence, long deleteRowCount) {
7580
return new DataFileMeta(
7681
name,
7782
maxKey - minKey + 1,
78-
1,
83+
maxKey - minKey + 1,
7984
row(minKey),
8085
row(maxKey),
8186
null,
@@ -84,7 +89,7 @@ public static DataFileMeta newFile(
8489
maxSequence,
8590
0,
8691
level,
87-
0L);
92+
deleteRowCount);
8893
}
8994

9095
public static BinaryRow row(int i) {

‎paimon-core/src/test/java/org/apache/paimon/table/source/SplitGeneratorTest.java

Lines changed: 80 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.paimon.data.InternalRow;
2222
import org.apache.paimon.io.DataFileMeta;
2323
import org.apache.paimon.table.BucketMode;
24+
import org.apache.paimon.utils.Pair;
2425

2526
import org.junit.jupiter.api.Test;
2627

@@ -31,8 +32,10 @@
3132
import java.util.stream.Collectors;
3233

3334
import static org.apache.paimon.CoreOptions.MergeEngine.DEDUPLICATE;
35+
import static org.apache.paimon.CoreOptions.MergeEngine.FIRST_ROW;
3436
import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
3537
import static org.apache.paimon.io.DataFileTestUtils.fromMinMax;
38+
import static org.apache.paimon.io.DataFileTestUtils.newFile;
3639
import static org.assertj.core.api.Assertions.assertThat;
3740

3841
/** Test for {@link AppendOnlySplitGenerator} and {@link MergeTreeSplitGenerator}. */
@@ -124,13 +127,86 @@ public void testMergeTree() {
124127
Collections.singletonList("6"));
125128
}
126129

127-
private List<List<String>> toNames(List<List<DataFileMeta>> splits) {
128-
return splits.stream()
130+
@Test
131+
public void testSplitRawConvertible() {
132+
Comparator<InternalRow> comparator = Comparator.comparingInt(o -> o.getInt(0));
133+
MergeTreeSplitGenerator mergeTreeSplitGenerator =
134+
new MergeTreeSplitGenerator(comparator, 100, 2, false, DEDUPLICATE);
135+
136+
// When level0 exists, should not be rawConvertible
137+
List<DataFileMeta> files1 =
138+
Arrays.asList(newFile("1", 0, 0, 10, 10L), newFile("2", 0, 10, 20, 20L));
139+
assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files1)))
140+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), false));
141+
142+
// When deleteRowCount > 0, should not be rawConvertible
143+
List<DataFileMeta> files2 =
144+
Arrays.asList(newFile("1", 1, 0, 10, 10L, 1L), newFile("2", 1, 10, 20, 20L));
145+
assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files2)))
146+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), false));
147+
148+
// No level0 and deleteRowCount == 0:
149+
// All in one level, should be rawConvertible
150+
List<DataFileMeta> files3 =
151+
Arrays.asList(newFile("1", 1, 0, 10, 10L), newFile("2", 1, 10, 20, 20L));
152+
assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files3)))
153+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), true));
154+
155+
// Not all in one level, should not be rawConvertible
156+
List<DataFileMeta> files4 =
157+
Arrays.asList(newFile("1", 1, 0, 10, 10L), newFile("2", 2, 10, 20, 20L));
158+
assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files4)))
159+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), false));
160+
161+
// Not all in one level but with deletion vectors enabled, should be rawConvertible
162+
MergeTreeSplitGenerator splitGeneratorWithDVEnabled =
163+
new MergeTreeSplitGenerator(comparator, 100, 2, true, DEDUPLICATE);
164+
assertThat(toNamesAndRawConvertible(splitGeneratorWithDVEnabled.splitForBatch(files4)))
165+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), true));
166+
167+
// Not all in one level but with first row merge engine, should be rawConvertible
168+
MergeTreeSplitGenerator splitGeneratorWithFirstRow =
169+
new MergeTreeSplitGenerator(comparator, 100, 2, false, FIRST_ROW);
170+
assertThat(toNamesAndRawConvertible(splitGeneratorWithFirstRow.splitForBatch(files4)))
171+
.containsExactlyInAnyOrder(Pair.of(Arrays.asList("1", "2"), true));
172+
173+
// Split with one file should be rawConvertible
174+
List<DataFileMeta> files5 =
175+
Arrays.asList(
176+
newFile("1", 1, 0, 10, 10L),
177+
newFile("2", 2, 0, 12, 12L),
178+
newFile("3", 3, 15, 60, 60L),
179+
newFile("4", 4, 18, 40, 40L),
180+
newFile("5", 5, 82, 85, 85L),
181+
newFile("6", 6, 100, 200, 200L));
182+
assertThat(toNamesAndRawConvertible(mergeTreeSplitGenerator.splitForBatch(files5)))
183+
.containsExactlyInAnyOrder(
184+
Pair.of(Arrays.asList("1", "2", "3", "4", "5"), false),
185+
Pair.of(Collections.singletonList("6"), true));
186+
}
187+
188+
private List<List<String>> toNames(List<SplitGenerator.SplitGroup> splitGroups) {
189+
return splitGroups.stream()
129190
.map(
130-
files ->
131-
files.stream()
191+
splitGroup ->
192+
splitGroup.files.stream()
132193
.map(DataFileMeta::fileName)
133194
.collect(Collectors.toList()))
134195
.collect(Collectors.toList());
135196
}
197+
198+
private List<Pair<List<String>, Boolean>> toNamesAndRawConvertible(
199+
List<SplitGenerator.SplitGroup> splitGroups) {
200+
return splitGroups.stream()
201+
.map(
202+
splitGroup -> {
203+
List<String> sortedFileNames =
204+
splitGroup.files.stream()
205+
.sorted(Comparator.comparing(DataFileMeta::fileName))
206+
.map(DataFileMeta::fileName)
207+
.collect(Collectors.toList());
208+
return Pair.of(sortedFileNames, splitGroup.rawConvertible);
209+
})
210+
.collect(Collectors.toList());
211+
}
136212
}

‎paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public void testGetPrimaryKeyRawFiles() throws Exception {
100100
assertThat(dataSplit.dataFiles()).hasSize(1);
101101
DataFileMeta meta = dataSplit.dataFiles().get(0);
102102
String partition = dataSplit.partition().getString(0).toString();
103-
assertThat(dataSplit.convertToRawFiles()).isNotPresent();
103+
assertThat(dataSplit.convertToRawFiles()).isPresent();
104104
}
105105

106106
// write another file on level 0

0 commit comments

Comments
 (0)