Skip to content

Commit

Permalink
Improved blob imports
Browse files Browse the repository at this point in the history
  • Loading branch information
@hudeany
hudeany committed Mar 3, 2024
1 parent 9868300 commit 07515c1
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 13 deletions.
Binary file modified lib_soderer/soderer-utilities-24.1.10.jar
View file
Binary file not shown.
9 changes: 4 additions & 5 deletions src/de/soderer/dbimport/DbImport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,6 @@ protected static int _main(final String[] args) {
}
wasAllowedParam = true;
} else if ("-noSingleMode".equalsIgnoreCase(arguments[i])) {
i++;
dbImportDefinition.setPreventBatchFallbackToSingleLineOnErrors(true);
wasAllowedParam = true;
} else {
Expand Down Expand Up @@ -598,8 +597,8 @@ protected static int _main(final String[] args) {
i++;
if (i >= arguments.length) {
throw new ParameterException(arguments[i - 1], "Missing parameter value for updatesql");
} else if (!arguments[i - 1].contains("?")) {
throw new ParameterException(arguments[i - 1], "Updatesql does not contain mandatory '?' placeholder");
} else if (!arguments[i].contains("?")) {
throw new ParameterException(arguments[i], "Updatesql does not contain mandatory '?' placeholder");
} else {
blobImportDefinition.setBlobImportStatement(arguments[i]);
}
Expand All @@ -608,8 +607,8 @@ protected static int _main(final String[] args) {
i++;
if (i >= arguments.length) {
throw new ParameterException(arguments[i - 1], "Missing parameter value for blobfile");
} else if (!new File(arguments[i - 1]).exists()) {
throw new ParameterException(arguments[i - 1], "Blob import file does not exist");
} else if (!new File(arguments[i]).exists()) {
throw new ParameterException(arguments[i], "Blob import file does not exist");
} else {
blobImportDefinition.setImportFilePath(arguments[i]);
}
Expand Down
26 changes: 24 additions & 2 deletions src/de/soderer/dbimport/DbImportWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -922,6 +922,13 @@ protected void insertIntoTable(final Connection connection, final String tableNa
try {
preparedStatement = connection.prepareStatement(statementString);

long maxBlobSize = -1;
if (dbDefinition.getDbVendor() == DbVendor.MySQL) {
maxBlobSize = DbUtilities.getMysqlConnectionNumericVariable(connection, "max_allowed_packet");
} else if (dbDefinition.getDbVendor() == DbVendor.MariaDB) {
maxBlobSize = DbUtilities.getMariaDBConnectionNumericVariable(connection, "max_allowed_packet");
}

final List<List<Object>> batchValues = new ArrayList<>();
Map<String, Object> itemData;
while ((itemData = dataProvider.getNextItemData()) != null) {
Expand All @@ -940,7 +947,7 @@ protected void insertIntoTable(final Connection connection, final String tableNa
final Object dataValue = itemData.get(mappingToUse.get(unescapedDbColumnToInsert).getFirst());
final String formatInfo = mappingToUse.get(unescapedDbColumnToInsert).getSecond();

final Closeable itemToClose = validateAndSetParameter(preparedStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry);
final Closeable itemToClose = validateAndSetParameter(preparedStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry, maxBlobSize);
if (itemToClose != null) {
itemsToCloseAfterwards.add(itemToClose);
}
Expand Down Expand Up @@ -1067,7 +1074,16 @@ protected void insertIntoTable(final Connection connection, final String tableNa
}
}

protected Closeable validateAndSetParameter(final PreparedStatement preparedStatement, final int columnIndex, final String columnName, final SimpleDataType simpleDataType, final boolean isNullable, final Object dataValue, final String formatInfo, final List<Object> batchValueItem) throws Exception {
protected Closeable validateAndSetParameter(
final PreparedStatement preparedStatement,
final int columnIndex,
final String columnName,
final SimpleDataType simpleDataType,
final boolean isNullable,
final Object dataValue,
final String formatInfo,
final List<Object> batchValueItem,
final long maxBlobSize) throws Exception {
Closeable itemToCloseAfterwards = null;
if (dataValue == null) {
if (!isNullable) {
Expand Down Expand Up @@ -1147,6 +1163,12 @@ protected Closeable validateAndSetParameter(final PreparedStatement preparedStat
} else if ("file".equalsIgnoreCase(formatInfo)) {
if (!new File(valueString).exists()) {
throw new DbImportException("File does not exist for column '" + columnName + "': " + valueString);
} else if (maxBlobSize >= 0 && maxBlobSize < new File(valueString).length()) {
if (dbDefinition.getDbVendor() == DbVendor.MySQL) {
throw new Exception("File size is too big for current database settings. Please adjust MySQL server variable 'max_allowed_packet' to at least " + new File(valueString).length());
} else if (dbDefinition.getDbVendor() == DbVendor.MariaDB) {
throw new Exception("File size is too big for current database settings. Please adjust MariaDB server variable 'max_allowed_packet' to at least " + new File(valueString).length());
}
} else if (simpleDataType == SimpleDataType.Blob) {
InputStream inputStream;
if (Utilities.endsWithIgnoreCase(valueString, ".zip")) {
Expand Down
10 changes: 5 additions & 5 deletions src/de/soderer/dbimport/DbNoSqlImportWorker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ protected void insertItemsWithoutKeyColumns(final Connection connection, final M
final String formatInfo = mappingToUse.get(unescapedDbColumnToInsert).getSecond();

final
Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry);
Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry, -1);
if (itemToClose != null) {
itemsToCloseAfterwards.add(itemToClose);
}
Expand Down Expand Up @@ -414,7 +414,7 @@ private void updateAndInsertItems(final Connection connection, final Map<String,
// Bug mitigation for Cassandra JDBC driver: Driver does not set apostrophes around strings as key column value in prepared statements
keyDataValue = "'" + keyDataValue + "'";
}
final Closeable keyItemToClose = validateAndSetParameter(preparedDetectStatement, keyIndex++, keyColumn, simpleDataType, dbColumns.get(keyColumn).isNullable(), keyDataValue, formatInfo, batchValueEntry);
final Closeable keyItemToClose = validateAndSetParameter(preparedDetectStatement, keyIndex++, keyColumn, simpleDataType, dbColumns.get(keyColumn).isNullable(), keyDataValue, formatInfo, batchValueEntry, -1);
if (keyItemToClose != null) {
detectItemsToCloseAfterwards.add(keyItemToClose);
}
Expand Down Expand Up @@ -467,7 +467,7 @@ private void updateAndInsertItems(final Connection connection, final Map<String,
final Object dataValue = itemData.get(mappingToUse.get(unescapedDbColumnToUpdate).getFirst());
final String formatInfo = mappingToUse.get(unescapedDbColumnToUpdate).getSecond();

final Closeable itemToClose = validateAndSetParameter(preparedUpdateStatement, i++, dbColumnToUpdate, simpleDataType, dbColumns.get(dbColumnToUpdate).isNullable(), dataValue, formatInfo, batchValueEntry);
final Closeable itemToClose = validateAndSetParameter(preparedUpdateStatement, i++, dbColumnToUpdate, simpleDataType, dbColumns.get(dbColumnToUpdate).isNullable(), dataValue, formatInfo, batchValueEntry, -1);
if (itemToClose != null) {
updateItemsToCloseAfterwards.add(itemToClose);
}
Expand All @@ -480,7 +480,7 @@ private void updateAndInsertItems(final Connection connection, final Map<String,
final Object keyDataValue = itemData.get(mappingToUse.get(unescapedDbKeyColumn).getFirst());
final String formatInfo = mappingToUse.get(unescapedDbKeyColumn).getSecond();

final Closeable itemToClose = validateAndSetParameter(preparedUpdateStatement, i++, keyColumn, simpleDataType, dbColumns.get(keyColumn).isNullable(), keyDataValue, formatInfo, batchValueEntry);
final Closeable itemToClose = validateAndSetParameter(preparedUpdateStatement, i++, keyColumn, simpleDataType, dbColumns.get(keyColumn).isNullable(), keyDataValue, formatInfo, batchValueEntry, -1);
if (itemToClose != null) {
detectItemsToCloseAfterwards.add(itemToClose);
}
Expand All @@ -504,7 +504,7 @@ private void updateAndInsertItems(final Connection connection, final Map<String,
final Object dataValue = itemData.get(mappingToUse.get(unescapedDbColumnToInsert).getFirst());
final String formatInfo = mappingToUse.get(unescapedDbColumnToInsert).getSecond();

final Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry);
final Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry, -1);
if (itemToClose != null) {
insertItemsToCloseAfterwards.add(itemToClose);
}
Expand Down
28 changes: 27 additions & 1 deletion test/de/soderer/dbimport/DbImportTest_MariaDB.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.Statement;
import java.util.Random;

import org.junit.After;
import org.junit.Assert;
Expand Down Expand Up @@ -33,7 +34,7 @@ public class DbImportTest_MariaDB {
public static final String USERNAME = System.getenv().get("USERNAME_MARIADB_TEST");
public static final String PASSWORD = System.getenv().get("PASSWORD_MARIADB_TEST");

public static final String[] DATA_TYPES = new String[] { "INTEGER", "DOUBLE", "VARCHAR(1024)", "BLOB", "LONGTEXT", "TIMESTAMP NULL", "DATE" };
public static final String[] DATA_TYPES = new String[] { "INTEGER", "DOUBLE", "VARCHAR(1024)", "LONGBLOB", "LONGTEXT", "TIMESTAMP NULL", "DATE" };

public static File INPUTFILE_CSV = new File(Utilities.replaceUsersHome("~" + File.separator + "temp" + File.separator + "test_tbl.csv"));
public static File INPUTFILE_JSON = new File(Utilities.replaceUsersHome("~" + File.separator + "temp" + File.separator + "test_tbl.json"));
Expand Down Expand Up @@ -91,6 +92,7 @@ private void createEmptyTestTable() throws Exception {
columnName = columnName.substring(0, columnName.indexOf("("));
}
columnName = columnName.replace("longtext", "clob");
columnName = columnName.replace("longblob", "blob");
columnName = columnName.replace("timestamp null", "timestamp");

if (dataColumnsPart.length() > 0) {
Expand Down Expand Up @@ -926,6 +928,30 @@ public void testCsvImportInsertBlobFile() {
}
}

@Test
public void testCsvImportInsertBigBlobFile() {
try {
createEmptyTestTable();
prefillTestTable();

final byte[] randomByteArray = new byte[20 * 1024 * 1024];
new Random().nextBytes(randomByteArray);
FileUtilities.write(BLOB_DATA_FILE, randomByteArray);

Assert.assertEquals(0, DbImport._main(new String[] {
"importblob",
"mariadb",
HOSTNAME,
DBNAME,
USERNAME,
"-updatesql", "UPDATE test_tbl SET column_blob = ? WHERE column_integer = 1",
"-blobfile", BLOB_DATA_FILE.getAbsolutePath(),
PASSWORD }));
} catch (final Exception e) {
Assert.fail(e.getMessage());
}
}

@Test
public void testCsvImportCreateTable() {
try {
Expand Down

0 comments on commit 07515c1

Please sign in to comment.