From 21629669245f9d7a6d96174edf3b8055c26a7a96 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 6 Apr 2020 13:14:34 +0200 Subject: [PATCH 1/3] Use InputStreamReader for serial UTF8 decoder The implementation is much more straightforward. It should also solve a JDK incompatiblity: java.lang.NoSuchMethodError: java.nio.ByteBuffer.flip()Ljava/nio/ByteBuffer; at processing.app.Serial.serialEvent(Serial.java:185) at jssc.SerialPort$LinuxEventThread.run(SerialPort.java:1299) See #8903 --- arduino-core/src/processing/app/Serial.java | 85 ++++++++------------- 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/arduino-core/src/processing/app/Serial.java b/arduino-core/src/processing/app/Serial.java index edc5e8f0c0f..75958b2d25f 100644 --- a/arduino-core/src/processing/app/Serial.java +++ b/arduino-core/src/processing/app/Serial.java @@ -22,23 +22,22 @@ package processing.app; -import jssc.SerialPort; -import jssc.SerialPortEvent; -import jssc.SerialPortEventListener; -import jssc.SerialPortException; +import static processing.app.I18n.format; +import static processing.app.I18n.tr; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; +import java.io.InputStreamReader; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.List; -import static processing.app.I18n.format; -import static processing.app.I18n.tr; +import jssc.SerialPort; +import jssc.SerialPortEvent; +import jssc.SerialPortEventListener; +import jssc.SerialPortException; public class Serial implements SerialPortEventListener { @@ -53,11 +52,8 @@ public class Serial implements SerialPortEventListener { private SerialPort port; - private CharsetDecoder bytesToStrings; - private static final int IN_BUFFER_CAPACITY = 128; - private static final int OUT_BUFFER_CAPACITY = 128; - private ByteBuffer inFromSerial = ByteBuffer.allocate(IN_BUFFER_CAPACITY); - private CharBuffer outToMessage = CharBuffer.allocate(OUT_BUFFER_CAPACITY); + private PipedOutputStream decoderInRaw; + private InputStreamReader decoderOutputUTF8; public Serial() throws SerialException { this(PreferencesData.get("serial.port"), @@ -189,42 +185,18 @@ public synchronized void serialEvent(SerialPortEvent serialEvent) { public void processSerialEvent(byte[] buf) { int next = 0; - // This uses a CharsetDecoder to convert from bytes to UTF-8 in - // a streaming fashion (i.e. where characters might be split - // over multiple reads). This needs the data to be in a - // ByteBuffer (inFromSerial, which we also use to store leftover - // incomplete characters for the nexst run) and produces a - // CharBuffer (outToMessage), which we then convert to char[] to - // pass onwards. - // Note that these buffers switch from input to output mode - // using flip/compact/clear - while (next < buf.length || inFromSerial.position() > 0) { - do { - // This might be 0 when all data was already read from buf - // (but then there will be data in inFromSerial left to - // decode). - int copyNow = Math.min(buf.length - next, inFromSerial.remaining()); - inFromSerial.put(buf, next, copyNow); - next += copyNow; - - inFromSerial.flip(); - bytesToStrings.decode(inFromSerial, outToMessage, false); - inFromSerial.compact(); - - // When there are multi-byte characters, outToMessage might - // still have room, so add more bytes if we have any. - } while (next < buf.length && outToMessage.hasRemaining()); - - // If no output was produced, the input only contained - // incomplete characters, so we're done processing - if (outToMessage.position() == 0) - break; - - outToMessage.flip(); - char[] chars = new char[outToMessage.remaining()]; - outToMessage.get(chars); - message(chars, chars.length); - outToMessage.clear(); + int max = buf.length; + char chars[] = new char[512]; + try { + while (next < max) { + int w = Integer.min(max - next, 128); + decoderInRaw.write(buf, next, w); + next += w; + int n = decoderOutputUTF8.read(chars); + message(chars, n); + } + } catch (IOException e) { + e.printStackTrace(); } } @@ -295,10 +267,13 @@ public void setRTS(boolean state) { * before they are handed as Strings to {@Link #message(char[], int)}. */ public synchronized void resetDecoding(Charset charset) { - bytesToStrings = charset.newDecoder() - .onMalformedInput(CodingErrorAction.REPLACE) - .onUnmappableCharacter(CodingErrorAction.REPLACE) - .replaceWith("\u2e2e"); + try { + decoderInRaw = new PipedOutputStream(); + decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw), charset); + } catch (IOException e) { + // Should never happen... + e.printStackTrace(); + } } static public List list() { From 618eef0e0db4a3e0d35318fde4d907b73b3d37ce Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 19 Jun 2020 16:05:56 +0200 Subject: [PATCH 2/3] Serial UTF-8 decoder now handles blocks of 16Kb at a time --- arduino-core/src/processing/app/Serial.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arduino-core/src/processing/app/Serial.java b/arduino-core/src/processing/app/Serial.java index 75958b2d25f..abeacf93a82 100644 --- a/arduino-core/src/processing/app/Serial.java +++ b/arduino-core/src/processing/app/Serial.java @@ -54,6 +54,7 @@ public class Serial implements SerialPortEventListener { private PipedOutputStream decoderInRaw; private InputStreamReader decoderOutputUTF8; + private final int DECODER_BUFF_SIZE = 16384; public Serial() throws SerialException { this(PreferencesData.get("serial.port"), @@ -186,10 +187,10 @@ public synchronized void serialEvent(SerialPortEvent serialEvent) { public void processSerialEvent(byte[] buf) { int next = 0; int max = buf.length; - char chars[] = new char[512]; + char chars[] = new char[DECODER_BUFF_SIZE]; try { while (next < max) { - int w = Integer.min(max - next, 128); + int w = Integer.min(max - next, chars.length); decoderInRaw.write(buf, next, w); next += w; int n = decoderOutputUTF8.read(chars); @@ -269,7 +270,8 @@ public void setRTS(boolean state) { public synchronized void resetDecoding(Charset charset) { try { decoderInRaw = new PipedOutputStream(); - decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw), charset); + // add 16 extra bytes to make room for incomplete UTF-8 chars + decoderOutputUTF8 = new InputStreamReader(new PipedInputStream(decoderInRaw, DECODER_BUFF_SIZE + 16), charset); } catch (IOException e) { // Should never happen... e.printStackTrace(); From 782a35bf9e4d64b612a21443c7671c53087dfafc Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 19 Jun 2020 16:06:18 +0200 Subject: [PATCH 3/3] Added test for invalid UTF-8 sequences --- app/test/processing/app/SerialTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/test/processing/app/SerialTest.java b/app/test/processing/app/SerialTest.java index 63280811e24..f63b8e99404 100644 --- a/app/test/processing/app/SerialTest.java +++ b/app/test/processing/app/SerialTest.java @@ -29,6 +29,7 @@ package processing.app; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import org.junit.Test; @@ -47,6 +48,16 @@ protected void message(char[] chars, int length) { String output = ""; } + @Test + public void testSerialUTF8DecoderWithInvalidChars() throws Exception { + NullSerial s = new NullSerial(); + byte[] testdata = new byte[] { '>', (byte) 0xC3, (byte) 0x28, '<' }; + byte[] expected = new byte[] { '>', (byte) 0xEF, (byte) 0xBF, (byte) 0xBD, (byte) 0x28, '<' }; + s.processSerialEvent(testdata); + byte[] res = s.output.getBytes("UTF-8"); + assertArrayEquals(expected, res); + } + @Test public void testSerialUTF8Decoder() throws Exception { NullSerial s = new NullSerial();