From d70bb0702b39ff9ac100687b2301622ecf4a8740 Mon Sep 17 00:00:00 2001 From: xujie Date: Tue, 22 Dec 2020 09:38:05 +0800 Subject: [PATCH 1/4] feat: parse the sdi page Signed-off-by: xujie --- .../innodb/java/reader/page/SdiPage.java | 76 ++++++++++++++++++ .../java/reader/page/index/SdiRecord.java | 79 +++++++++++++++++++ .../innodb/java/reader/util/ZlibUtil.java | 42 ++++++++++ 3 files changed, 197 insertions(+) create mode 100644 innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java create mode 100644 innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java index 1bb2661..4748fc2 100644 --- a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java @@ -3,6 +3,19 @@ */ package com.alibaba.innodb.java.reader.page; +import com.alibaba.innodb.java.reader.exception.ReaderException; +import com.alibaba.innodb.java.reader.page.index.*; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import static com.alibaba.innodb.java.reader.Constants.BYTES_OF_INFIMUM; +import static com.alibaba.innodb.java.reader.Constants.BYTES_OF_SUPREMUM; +import static com.alibaba.innodb.java.reader.SizeOf.*; +import static com.alibaba.innodb.java.reader.SizeOf.SIZE_OF_PAGE_DIR_SLOT; +import static com.google.common.base.Preconditions.checkState; + /** * Since MySQL8.0, there is SDI, a.k.a Serialized Dictionary Information(SDI). * @@ -10,8 +23,71 @@ */ public class SdiPage extends AbstractPage { + private IndexHeader indexHeader; + + private FsegHeader fsegHeader; + + private RecordHeader infimumHeader; + + private RecordHeader supremumHeader; + + private int[] dirSlots; + + private List sdiRecordList = new LinkedList<>(); + public SdiPage(InnerPage innerPage) { super(innerPage); + this.indexHeader = IndexHeader.fromSlice(sliceInput); + if (this.indexHeader.getFormat() != PageFormat.COMPACT) { + reportException(); + } + this.fsegHeader = FsegHeader.fromSlice(sliceInput); + + this.infimumHeader = RecordHeader.fromSlice(sliceInput); + checkState(Arrays.equals(sliceInput.readByteArray(SIZE_OF_MUM_RECORD), BYTES_OF_INFIMUM)); + + this.supremumHeader = RecordHeader.fromSlice(sliceInput); + checkState(Arrays.equals(sliceInput.readByteArray(SIZE_OF_MUM_RECORD), BYTES_OF_SUPREMUM)); + + int endOfSupremum = sliceInput.position(); + int dirSlotNum = this.indexHeader.getNumOfDirSlots(); + dirSlots = new int[dirSlotNum]; + sliceInput.setPosition(SIZE_OF_PAGE - SIZE_OF_FIL_TRAILER - dirSlotNum * SIZE_OF_PAGE_DIR_SLOT); + for (int i = 0; i < dirSlotNum; i++) { + dirSlots[dirSlotNum - i - 1] = sliceInput.readUnsignedShort(); + } + sliceInput.setPosition(dirSlots[0]); + RecordHeader recordHeader = null; + while (true) { + sliceInput.decrPosition(5); + recordHeader = RecordHeader.fromSlice(sliceInput); + if (recordHeader.getRecordType() == RecordType.INFIMUM) { + sliceInput.setPosition(sliceInput.position() + recordHeader.getNextRecOffset()); + continue; + } + if (recordHeader.getRecordType() == RecordType.SUPREMUM) { + break; + } + try { + int pos = sliceInput.position(); + SdiRecord record = new SdiRecord(sliceInput, recordHeader); + this.sdiRecordList.add(record); + sliceInput.setPosition(pos + recordHeader.getNextRecOffset()); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + } + + private void reportException() throws ReaderException { + if (this.indexHeader.getIndexId() <= 0L + && this.indexHeader.getMaxTrxId() <= 0L) { + throw new ReaderException("Index header is unreadable, only new-style compact page format is supported, " + + "please make sure the file is a valid InnoDB data file, page=" + + innerPage.toString() + ", index.header = " + this.indexHeader.toString()); + } + throw new ReaderException("Only new-style compact page format is supported, page=" + innerPage.toString() + + ", index.header = " + this.indexHeader.toString()); } } diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java new file mode 100644 index 0000000..ef7a667 --- /dev/null +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java @@ -0,0 +1,79 @@ +package com.alibaba.innodb.java.reader.page.index; + +import com.alibaba.innodb.java.reader.util.SliceInput; +import com.alibaba.innodb.java.reader.util.ZlibUtil; +import lombok.Data; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * @author jie xu + * @description + * @created 2020/12/21 + **/ +@Data +public class SdiRecord { + + private int contentLength; + + private RecordHeader recordHeader; + + private int sdiType; + + private long sdiId; + + private long sdiTransId; + + private int unzipLength; + + private int zipLength; + + private String content; + + public SdiRecord(SliceInput sliceInput, RecordHeader recordHeader) throws IOException { + this.recordHeader = recordHeader; + int position = sliceInput.position(); + this.contentLength = parseContentLength(sliceInput); + sliceInput.setPosition(position); + this.sdiType = sliceInput.readInt(); + this.sdiId = sliceInput.readLong(); + this.sdiTransId = sliceInput.read6BytesInt(); + //skip rollBackRef + sliceInput.setPosition(sliceInput.position() + 7); + this.unzipLength = sliceInput.readInt(); + this.zipLength = sliceInput.readInt(); + if (this.contentLength != zipLength && this.contentLength != unzipLength) { + throw new RuntimeException("sdi record parse error!"); + } + byte[] buffer = new byte[this.contentLength]; + sliceInput.readBytes(buffer); + if (this.contentLength == zipLength) { + buffer = ZlibUtil.decompress(buffer); + } + this.content = new String(buffer, 0, buffer.length, StandardCharsets.US_ASCII); + sliceInput.setPosition(position); + } + + private int parseContentLength(SliceInput sliceInput) { + int resu = 0; + sliceInput.decrPosition(6); + int first = sliceInput.readUnsignedByte(); + if ((first & 0x80) != 0) { + resu = (first & 0x3f) << 8; + if ((first & 0x40) != 0x00) { + //todo big data length parse + throw new RuntimeException("unsupported length"); + } else { + sliceInput.decrPosition(2); + int second = sliceInput.readUnsignedByte(); + sliceInput.readUnsignedByte(); + resu += second; + } + } else { + resu = first; + } + return resu; + } + +} diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java new file mode 100644 index 0000000..8f23f72 --- /dev/null +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java @@ -0,0 +1,42 @@ +package com.alibaba.innodb.java.reader.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.Inflater; + +/** + * @author jie xu + * @description + * @created 2020/12/22 + **/ +public class ZlibUtil { + public static byte[] decompress(byte[] data) { + byte[] output = new byte[0]; + + Inflater decompresser = new Inflater(); + decompresser.reset(); + decompresser.setInput(data); + + ByteArrayOutputStream o = new ByteArrayOutputStream(data.length); + try { + byte[] buf = new byte[1024]; + while (!decompresser.finished()) { + int i = decompresser.inflate(buf); + o.write(buf, 0, i); + } + output = o.toByteArray(); + } catch (Exception e) { + output = data; + e.printStackTrace(); + } finally { + try { + o.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + decompresser.end(); + return output; + } +} From ec359c9369f5a0ce6388983cdc71631fbec546c2 Mon Sep 17 00:00:00 2001 From: xujie Date: Thu, 24 Dec 2020 18:14:10 +0800 Subject: [PATCH 2/4] refactor: change magic number to const val --- .../alibaba/innodb/java/reader/page/SdiPage.java | 15 +++++---------- .../innodb/java/reader/page/index/SdiRecord.java | 11 ++++++----- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java index 4748fc2..1facfa3 100644 --- a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java @@ -49,7 +49,6 @@ public SdiPage(InnerPage innerPage) { this.supremumHeader = RecordHeader.fromSlice(sliceInput); checkState(Arrays.equals(sliceInput.readByteArray(SIZE_OF_MUM_RECORD), BYTES_OF_SUPREMUM)); - int endOfSupremum = sliceInput.position(); int dirSlotNum = this.indexHeader.getNumOfDirSlots(); dirSlots = new int[dirSlotNum]; sliceInput.setPosition(SIZE_OF_PAGE - SIZE_OF_FIL_TRAILER - dirSlotNum * SIZE_OF_PAGE_DIR_SLOT); @@ -59,7 +58,7 @@ public SdiPage(InnerPage innerPage) { sliceInput.setPosition(dirSlots[0]); RecordHeader recordHeader = null; while (true) { - sliceInput.decrPosition(5); + sliceInput.decrPosition(SIZE_OF_REC_HEADER); recordHeader = RecordHeader.fromSlice(sliceInput); if (recordHeader.getRecordType() == RecordType.INFIMUM) { sliceInput.setPosition(sliceInput.position() + recordHeader.getNextRecOffset()); @@ -68,14 +67,10 @@ public SdiPage(InnerPage innerPage) { if (recordHeader.getRecordType() == RecordType.SUPREMUM) { break; } - try { - int pos = sliceInput.position(); - SdiRecord record = new SdiRecord(sliceInput, recordHeader); - this.sdiRecordList.add(record); - sliceInput.setPosition(pos + recordHeader.getNextRecOffset()); - } catch (Exception ex) { - throw new RuntimeException(ex); - } + int pos = sliceInput.position(); + SdiRecord record = new SdiRecord(sliceInput, recordHeader); + this.sdiRecordList.add(record); + sliceInput.setPosition(pos + recordHeader.getNextRecOffset()); } } diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java index ef7a667..4499c95 100644 --- a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/index/SdiRecord.java @@ -4,9 +4,10 @@ import com.alibaba.innodb.java.reader.util.ZlibUtil; import lombok.Data; -import java.io.IOException; import java.nio.charset.StandardCharsets; +import static com.alibaba.innodb.java.reader.SizeOf.SIZE_OF_REC_HEADER; + /** * @author jie xu * @description @@ -31,7 +32,7 @@ public class SdiRecord { private String content; - public SdiRecord(SliceInput sliceInput, RecordHeader recordHeader) throws IOException { + public SdiRecord(SliceInput sliceInput, RecordHeader recordHeader) { this.recordHeader = recordHeader; int position = sliceInput.position(); this.contentLength = parseContentLength(sliceInput); @@ -57,13 +58,13 @@ public SdiRecord(SliceInput sliceInput, RecordHeader recordHeader) throws IOExce private int parseContentLength(SliceInput sliceInput) { int resu = 0; - sliceInput.decrPosition(6); + sliceInput.decrPosition(SIZE_OF_REC_HEADER + 1); int first = sliceInput.readUnsignedByte(); if ((first & 0x80) != 0) { resu = (first & 0x3f) << 8; if ((first & 0x40) != 0x00) { - //todo big data length parse - throw new RuntimeException("unsupported length"); + //todo big data length, read from overflowPage (BLOB page?) + throw new RuntimeException("unsupported record length"); } else { sliceInput.decrPosition(2); int second = sliceInput.readUnsignedByte(); From 8caa32bff94479324147063d304ade92d01c926e Mon Sep 17 00:00:00 2001 From: xujie Date: Thu, 24 Dec 2020 18:34:09 +0800 Subject: [PATCH 3/4] refactor: delete repeat code --- .../alibaba/innodb/java/reader/page/SdiPage.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java index 1facfa3..d31c360 100644 --- a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/page/SdiPage.java @@ -39,7 +39,7 @@ public SdiPage(InnerPage innerPage) { super(innerPage); this.indexHeader = IndexHeader.fromSlice(sliceInput); if (this.indexHeader.getFormat() != PageFormat.COMPACT) { - reportException(); + throw new ReaderException("only support COMPACT format"); } this.fsegHeader = FsegHeader.fromSlice(sliceInput); @@ -74,15 +74,4 @@ public SdiPage(InnerPage innerPage) { } } - private void reportException() throws ReaderException { - if (this.indexHeader.getIndexId() <= 0L - && this.indexHeader.getMaxTrxId() <= 0L) { - throw new ReaderException("Index header is unreadable, only new-style compact page format is supported, " - + "please make sure the file is a valid InnoDB data file, page=" - + innerPage.toString() + ", index.header = " + this.indexHeader.toString()); - } - throw new ReaderException("Only new-style compact page format is supported, page=" + innerPage.toString() - + ", index.header = " + this.indexHeader.toString()); - } - } From 12a5072d947f0818f7c3869d8dc04af522900eea Mon Sep 17 00:00:00 2001 From: xujie Date: Mon, 28 Dec 2020 19:42:23 +0800 Subject: [PATCH 4/4] refactor: throw runtime exception when decompress error --- .../innodb/java/reader/util/ZlibUtil.java | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java index 8f23f72..ac16bf5 100644 --- a/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java +++ b/innodb-java-reader/src/main/java/com/alibaba/innodb/java/reader/util/ZlibUtil.java @@ -10,33 +10,28 @@ * @created 2020/12/22 **/ public class ZlibUtil { + private static final int BUF_SIZE = 1024; public static byte[] decompress(byte[] data) { - byte[] output = new byte[0]; - Inflater decompresser = new Inflater(); decompresser.reset(); decompresser.setInput(data); - ByteArrayOutputStream o = new ByteArrayOutputStream(data.length); try { - byte[] buf = new byte[1024]; + byte[] buf = new byte[BUF_SIZE]; while (!decompresser.finished()) { int i = decompresser.inflate(buf); o.write(buf, 0, i); } - output = o.toByteArray(); + return o.toByteArray(); } catch (Exception e) { - output = data; - e.printStackTrace(); + throw new RuntimeException(e); } finally { try { + decompresser.end(); o.close(); - } catch (IOException e) { - e.printStackTrace(); + } catch (IOException ignored) { + } } - - decompresser.end(); - return output; } }