Skip to content

Commit

Permalink
MySQL client data type mapping rework - fixes #1239
Browse files Browse the repository at this point in the history
  • Loading branch information
@BillyYccc @vietj
BillyYccc authored and vietj committed Jul 22, 2023
1 parent f6d1376 commit fffcad4
Show file tree
Hide file tree
Showing 9 changed files with 427 additions and 195 deletions.
6 changes: 4 additions & 2 deletions vertx-mysql-client/src/main/java/io/vertx/mysqlclient/impl/codec/CommandCodec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,15 +176,17 @@ ColumnDefinition decodeColumnDefinitionPacketPayload(ByteBuf payload) {
int characterSet = payload.getUnsignedShortLE(start + bytesToSkip);
bytesToSkip += 6; // characterSet + columnLength

DataType type = DataType.valueOf(payload.getUnsignedByte(start + bytesToSkip));
short type = payload.getUnsignedByte(start + bytesToSkip);
bytesToSkip++;

int flags = payload.getUnsignedShortLE(start + bytesToSkip);
bytesToSkip += 2; // flags + decimals

payload.skipBytes(bytesToSkip);

return new ColumnDefinition(name, characterSet, type, flags);
// convert type+characterset+flags to dataType
DataType dataType = DataType.parseDataType(type, characterSet, flags);
return new ColumnDefinition(name, characterSet, dataType, flags);
}

void skipEofPacketIfNeeded(ByteBuf payload) {
Expand Down
2 changes: 1 addition & 1 deletion ...l-client/src/main/java/io/vertx/mysqlclient/impl/codec/ExtendedQueryCommandBaseCodec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ protected final void sendStatementExecuteCommand(MySQLPreparedStatement statemen

if (sendTypesToServer) {
for (DataType bindingType : statement.bindingTypes()) {
packet.writeByte(bindingType.id);
packet.writeByte(bindingType.getColumnType());
packet.writeByte(0); // parameter flag: signed
}
}
Expand Down
6 changes: 2 additions & 4 deletions vertx-mysql-client/src/main/java/io/vertx/mysqlclient/impl/codec/RowResultDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,7 @@ protected Row decodeRow(int len, ByteBuf in) {
ColumnDefinition columnDef = rowDesc.get(c);
DataType dataType = columnDef.type();
int collationId = columnDef.characterSet();
int columnDefinitionFlags = columnDef.flags();
decoded = DataTypeCodec.decodeBinary(dataType, collationId, columnDefinitionFlags, in);
decoded = DataTypeCodec.decodeBinary(dataType, collationId, in);
}
row.addValue(decoded);
}
Expand All @@ -71,9 +70,8 @@ protected Row decodeRow(int len, ByteBuf in) {
} else {
ColumnDefinition columnDef = rowDesc.get(c);
DataType dataType = columnDef.type();
int columnDefinitionFlags = columnDef.flags();
int collationId = columnDef.characterSet();
decoded = DataTypeCodec.decodeText(dataType, collationId, columnDefinitionFlags, in);
decoded = DataTypeCodec.decodeText(dataType, collationId, in);
}
row.addValue(decoded);
}
Expand Down
172 changes: 129 additions & 43 deletions vertx-mysql-client/src/main/java/io/vertx/mysqlclient/impl/datatype/DataType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,75 +11,161 @@

package io.vertx.mysqlclient.impl.datatype;

import io.netty.util.collection.IntObjectHashMap;
import io.netty.util.collection.IntObjectMap;
import io.netty.util.collection.ShortObjectHashMap;
import io.netty.util.collection.ShortObjectMap;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.impl.logging.Logger;
import io.vertx.core.impl.logging.LoggerFactory;
import io.vertx.mysqlclient.data.spatial.Geometry;
import io.vertx.mysqlclient.impl.MySQLCollation;
import io.vertx.mysqlclient.impl.protocol.ColumnDefinition;
import io.vertx.sqlclient.data.Numeric;

import java.math.BigInteger;
import java.sql.JDBCType;
import java.time.Duration;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.List;

public enum DataType {
INT1(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY, Byte.class, Byte.class, JDBCType.TINYINT),
INT2(ColumnDefinition.ColumnType.MYSQL_TYPE_SHORT, Short.class, Short.class, JDBCType.SMALLINT),
INT3(ColumnDefinition.ColumnType.MYSQL_TYPE_INT24, Integer.class, Integer.class, JDBCType.INTEGER),
INT4(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG, Integer.class, Integer.class, JDBCType.INTEGER),
INT8(ColumnDefinition.ColumnType.MYSQL_TYPE_LONGLONG, Long.class, Long.class, JDBCType.BIGINT),
DOUBLE(ColumnDefinition.ColumnType.MYSQL_TYPE_DOUBLE, Double.class, Double.class, JDBCType.DOUBLE),
FLOAT(ColumnDefinition.ColumnType.MYSQL_TYPE_FLOAT, Float.class, Float.class, JDBCType.REAL),
NUMERIC(ColumnDefinition.ColumnType.MYSQL_TYPE_NEWDECIMAL, Numeric.class, Numeric.class, JDBCType.DECIMAL), // DECIMAL
STRING(ColumnDefinition.ColumnType.MYSQL_TYPE_STRING, Buffer.class, String.class, JDBCType.VARCHAR), // CHAR, BINARY
VARSTRING(ColumnDefinition.ColumnType.MYSQL_TYPE_VAR_STRING, Buffer.class, String.class, JDBCType.VARCHAR), //VARCHAR, VARBINARY
TINYBLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY_BLOB, Buffer.class, String.class, JDBCType.BLOB),
BLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_BLOB, Buffer.class, String.class, JDBCType.BLOB),
MEDIUMBLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_MEDIUM_BLOB, Buffer.class, String.class, JDBCType.BLOB),
LONGBLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG_BLOB, Buffer.class, String.class, JDBCType.BLOB),
DATE(ColumnDefinition.ColumnType.MYSQL_TYPE_DATE, LocalDate.class, LocalDate.class, JDBCType.DATE),
TIME(ColumnDefinition.ColumnType.MYSQL_TYPE_TIME, Duration.class, Duration.class, JDBCType.TIME),
DATETIME(ColumnDefinition.ColumnType.MYSQL_TYPE_DATETIME, LocalDateTime.class, LocalDateTime.class, JDBCType.TIMESTAMP),
YEAR(ColumnDefinition.ColumnType.MYSQL_TYPE_YEAR, Short.class, Short.class, JDBCType.SMALLINT),
TIMESTAMP(ColumnDefinition.ColumnType.MYSQL_TYPE_TIMESTAMP, LocalDateTime.class, LocalDateTime.class, JDBCType.TIMESTAMP),
BIT(ColumnDefinition.ColumnType.MYSQL_TYPE_BIT, Long.class, Long.class, JDBCType.BIT),
JSON(ColumnDefinition.ColumnType.MYSQL_TYPE_JSON, Object.class, Object.class, JDBCType.OTHER),
GEOMETRY(ColumnDefinition.ColumnType.MYSQL_TYPE_GEOMETRY, Geometry.class, Geometry.class, JDBCType.OTHER),
NULL(ColumnDefinition.ColumnType.MYSQL_TYPE_NULL, Object.class, Object.class, JDBCType.OTHER), // useful for mariadb prepare statement response
UNBIND(-1, Object.class, Object.class, JDBCType.OTHER); // useful for binding param values

// unsigned int
U_INT8(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY, Short.class, JDBCType.SMALLINT),
U_INT16(ColumnDefinition.ColumnType.MYSQL_TYPE_SHORT, Integer.class, JDBCType.INTEGER),
U_INT24(ColumnDefinition.ColumnType.MYSQL_TYPE_INT24, Integer.class, JDBCType.INTEGER),
U_INT32(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG, Long.class, JDBCType.BIGINT),
U_INT64(ColumnDefinition.ColumnType.MYSQL_TYPE_LONGLONG, BigInteger.class, JDBCType.NUMERIC),
// signed int
INT8(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY, Byte.class, JDBCType.TINYINT),
INT16(ColumnDefinition.ColumnType.MYSQL_TYPE_SHORT, Short.class, JDBCType.SMALLINT),
INT24(ColumnDefinition.ColumnType.MYSQL_TYPE_INT24, Integer.class, JDBCType.INTEGER),
INT32(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG, Integer.class, JDBCType.INTEGER),
INT64(ColumnDefinition.ColumnType.MYSQL_TYPE_LONGLONG, Long.class, JDBCType.BIGINT),
// numeric
BIT(ColumnDefinition.ColumnType.MYSQL_TYPE_BIT, Long.class, JDBCType.BIT),
DOUBLE(ColumnDefinition.ColumnType.MYSQL_TYPE_DOUBLE, Double.class, JDBCType.DOUBLE),
FLOAT(ColumnDefinition.ColumnType.MYSQL_TYPE_FLOAT, Float.class, JDBCType.FLOAT),
NUMERIC(ColumnDefinition.ColumnType.MYSQL_TYPE_NEWDECIMAL, Numeric.class, JDBCType.DECIMAL),
// string
STRING(ColumnDefinition.ColumnType.MYSQL_TYPE_STRING, String.class, JDBCType.CHAR),
VARSTRING(ColumnDefinition.ColumnType.MYSQL_TYPE_VAR_STRING, String.class, JDBCType.VARCHAR),
// clob
TINY_TEXT(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY_BLOB, String.class, JDBCType.CLOB),
TEXT(ColumnDefinition.ColumnType.MYSQL_TYPE_BLOB, String.class, JDBCType.CLOB),
MEDIUM_TEXT(ColumnDefinition.ColumnType.MYSQL_TYPE_MEDIUM_BLOB, String.class, JDBCType.CLOB),
LONG_TEXT(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG_BLOB, String.class, JDBCType.CLOB),
// binary
BINARY(ColumnDefinition.ColumnType.MYSQL_TYPE_STRING, Buffer.class, JDBCType.BINARY),
VARBINARY(ColumnDefinition.ColumnType.MYSQL_TYPE_VAR_STRING, Buffer.class, JDBCType.VARBINARY),
// blob
TINY_BLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY_BLOB, Buffer.class, JDBCType.BLOB),
BLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_BLOB, Buffer.class, JDBCType.BLOB),
MEDIUM_BLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_MEDIUM_BLOB, Buffer.class, JDBCType.BLOB),
LONG_BLOB(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG_BLOB, Buffer.class, JDBCType.BLOB),
// time
DATE(ColumnDefinition.ColumnType.MYSQL_TYPE_DATE, LocalDate.class, JDBCType.DATE),
TIME(ColumnDefinition.ColumnType.MYSQL_TYPE_TIME, Duration.class, JDBCType.TIME),
DATETIME(ColumnDefinition.ColumnType.MYSQL_TYPE_DATETIME, LocalDateTime.class, JDBCType.TIMESTAMP),
YEAR(ColumnDefinition.ColumnType.MYSQL_TYPE_YEAR, Short.class, JDBCType.SMALLINT),
TIMESTAMP(ColumnDefinition.ColumnType.MYSQL_TYPE_TIMESTAMP, LocalDateTime.class, JDBCType.TIMESTAMP),

JSON(ColumnDefinition.ColumnType.MYSQL_TYPE_JSON, Object.class, JDBCType.OTHER),
GEOMETRY(ColumnDefinition.ColumnType.MYSQL_TYPE_GEOMETRY, Geometry.class, JDBCType.OTHER),
NULL(ColumnDefinition.ColumnType.MYSQL_TYPE_NULL, Object.class, JDBCType.OTHER), // useful for mariadb prepare statement response
UNBIND((short) -1, Object.class, JDBCType.OTHER); // useful for binding param values
;

private static final Logger LOGGER = LoggerFactory.getLogger(DataType.class);

private static IntObjectMap<DataType> idToDataType = new IntObjectHashMap<>();
// protocol id
private final short columnType;

static {
for (DataType dataType : values()) {
idToDataType.put(dataType.id, dataType);
}
}
private final Class<?> javaType;

public final int id;
public final Class<?> binaryType;
public final Class<?> textType;
public final JDBCType jdbcType;
private final JDBCType jdbcType;

DataType(int id, Class<?> binaryType, Class<?> textType, JDBCType jdbcType) {
this.id = id;
this.binaryType = binaryType;
this.textType = textType;
DataType(short columnType, Class<?> javaType, JDBCType jdbcType) {
this.columnType = columnType;
this.javaType = javaType;
this.jdbcType = jdbcType;
}

public static DataType valueOf(int value) {
DataType dataType = idToDataType.get(value);
public int getColumnType() {
return columnType;
}

public Class<?> getJavaType() {
return javaType;
}

public JDBCType getJdbcType() {
return jdbcType;
}

public static DataType parseDataType(short type, int characterSet, int flags) {
// integer
if (COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.containsKey(type)) {
return isUnsignedNumeric(flags) ? COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.get(type).get(0) : COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.get(type).get(1);
}

// string
if (COLUMN_TYPE_TO_STRING_TYPE_MAPPING.containsKey(type)) {
return isText(characterSet) ? COLUMN_TYPE_TO_STRING_TYPE_MAPPING.get(type).get(0) : COLUMN_TYPE_TO_STRING_TYPE_MAPPING.get(type).get(1);
}

// others
DataType dataType = COLUMN_TYPE_TO_DATA_TYPE_MAPPING.get(type);
if (dataType == null) {
LOGGER.warn(String.format("MySQL data type Id =[%d] not handled - using string type instead", value));
LOGGER.warn(String.format("MySQL data type Id =[%d] not handled - using string type instead", type));
return STRING;
} else {
return dataType;
}
}

private static final ShortObjectMap<List<DataType>> COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING;
private static final ShortObjectMap<List<DataType>> COLUMN_TYPE_TO_STRING_TYPE_MAPPING;
private static final ShortObjectMap<DataType> COLUMN_TYPE_TO_DATA_TYPE_MAPPING;

static {
// integer
// column type -> uint, int
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING = new ShortObjectHashMap<>(5, 1.0f);
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY, Arrays.asList(U_INT8, INT8));
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_SHORT, Arrays.asList(U_INT16, INT16));
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_INT24, Arrays.asList(U_INT24, INT24));
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG, Arrays.asList(U_INT32, INT32));
COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_LONGLONG, Arrays.asList(U_INT64, INT64));

// string
// column type -> clob, blob
COLUMN_TYPE_TO_STRING_TYPE_MAPPING = new ShortObjectHashMap<>(6, 1.0f);
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_STRING, Arrays.asList(STRING, BINARY));
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_VAR_STRING, Arrays.asList(VARSTRING, VARBINARY));
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_TINY_BLOB, Arrays.asList(TINY_TEXT, TINY_BLOB));
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_BLOB, Arrays.asList(TEXT, BLOB));
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_MEDIUM_BLOB, Arrays.asList(MEDIUM_TEXT, MEDIUM_BLOB));
COLUMN_TYPE_TO_STRING_TYPE_MAPPING.put(ColumnDefinition.ColumnType.MYSQL_TYPE_LONG_BLOB, Arrays.asList(LONG_TEXT, LONG_BLOB));

// others
COLUMN_TYPE_TO_DATA_TYPE_MAPPING = new ShortObjectHashMap<>(13, 1.0f);
for (DataType dataType : DataType.values()) {
if (COLUMN_TYPE_TO_INTEGER_TYPE_MAPPING.containsKey(dataType.columnType) || COLUMN_TYPE_TO_STRING_TYPE_MAPPING.containsKey(dataType.columnType)) {
continue;
}

COLUMN_TYPE_TO_DATA_TYPE_MAPPING.put(dataType.columnType, dataType);
}
}

private static boolean isText(int collationId) {
return collationId != MySQLCollation.binary.collationId();
}

private static boolean isUnsignedNumeric(int columnDefinitionFlags) {
return (columnDefinitionFlags & ColumnDefinition.ColumnDefinitionFlags.UNSIGNED_FLAG) != 0;
}

}
Loading

0 comments on commit fffcad4

Please sign in to comment.