diff --git a/lib_soderer/soderer-utilities-24.1.10.jar b/lib_soderer/soderer-utilities-24.1.10.jar index 5e04bf6..97b3af8 100644 Binary files a/lib_soderer/soderer-utilities-24.1.10.jar and b/lib_soderer/soderer-utilities-24.1.10.jar differ diff --git a/src/de/soderer/dbimport/DbImport.java b/src/de/soderer/dbimport/DbImport.java index b0d4bb7..501fc2d 100644 --- a/src/de/soderer/dbimport/DbImport.java +++ b/src/de/soderer/dbimport/DbImport.java @@ -570,7 +570,6 @@ protected static int _main(final String[] args) { } wasAllowedParam = true; } else if ("-noSingleMode".equalsIgnoreCase(arguments[i])) { - i++; dbImportDefinition.setPreventBatchFallbackToSingleLineOnErrors(true); wasAllowedParam = true; } else { @@ -598,8 +597,8 @@ protected static int _main(final String[] args) { i++; if (i >= arguments.length) { throw new ParameterException(arguments[i - 1], "Missing parameter value for updatesql"); - } else if (!arguments[i - 1].contains("?")) { - throw new ParameterException(arguments[i - 1], "Updatesql does not contain mandatory '?' placeholder"); + } else if (!arguments[i].contains("?")) { + throw new ParameterException(arguments[i], "Updatesql does not contain mandatory '?' placeholder"); } else { blobImportDefinition.setBlobImportStatement(arguments[i]); } @@ -608,8 +607,8 @@ protected static int _main(final String[] args) { i++; if (i >= arguments.length) { throw new ParameterException(arguments[i - 1], "Missing parameter value for blobfile"); - } else if (!new File(arguments[i - 1]).exists()) { - throw new ParameterException(arguments[i - 1], "Blob import file does not exist"); + } else if (!new File(arguments[i]).exists()) { + throw new ParameterException(arguments[i], "Blob import file does not exist"); } else { blobImportDefinition.setImportFilePath(arguments[i]); } diff --git a/src/de/soderer/dbimport/DbImportWorker.java b/src/de/soderer/dbimport/DbImportWorker.java index 8833853..8bcfa4b 100644 --- a/src/de/soderer/dbimport/DbImportWorker.java +++ b/src/de/soderer/dbimport/DbImportWorker.java @@ -922,6 +922,13 @@ protected void insertIntoTable(final Connection connection, final String tableNa try { preparedStatement = connection.prepareStatement(statementString); + long maxBlobSize = -1; + if (dbDefinition.getDbVendor() == DbVendor.MySQL) { + maxBlobSize = DbUtilities.getMysqlConnectionNumericVariable(connection, "max_allowed_packet"); + } else if (dbDefinition.getDbVendor() == DbVendor.MariaDB) { + maxBlobSize = DbUtilities.getMariaDBConnectionNumericVariable(connection, "max_allowed_packet"); + } + final List> batchValues = new ArrayList<>(); Map itemData; while ((itemData = dataProvider.getNextItemData()) != null) { @@ -940,7 +947,7 @@ protected void insertIntoTable(final Connection connection, final String tableNa final Object dataValue = itemData.get(mappingToUse.get(unescapedDbColumnToInsert).getFirst()); final String formatInfo = mappingToUse.get(unescapedDbColumnToInsert).getSecond(); - final Closeable itemToClose = validateAndSetParameter(preparedStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry); + final Closeable itemToClose = validateAndSetParameter(preparedStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry, maxBlobSize); if (itemToClose != null) { itemsToCloseAfterwards.add(itemToClose); } @@ -1067,7 +1074,16 @@ protected void insertIntoTable(final Connection connection, final String tableNa } } - protected Closeable validateAndSetParameter(final PreparedStatement preparedStatement, final int columnIndex, final String columnName, final SimpleDataType simpleDataType, final boolean isNullable, final Object dataValue, final String formatInfo, final List batchValueItem) throws Exception { + protected Closeable validateAndSetParameter( + final PreparedStatement preparedStatement, + final int columnIndex, + final String columnName, + final SimpleDataType simpleDataType, + final boolean isNullable, + final Object dataValue, + final String formatInfo, + final List batchValueItem, + final long maxBlobSize) throws Exception { Closeable itemToCloseAfterwards = null; if (dataValue == null) { if (!isNullable) { @@ -1147,6 +1163,12 @@ protected Closeable validateAndSetParameter(final PreparedStatement preparedStat } else if ("file".equalsIgnoreCase(formatInfo)) { if (!new File(valueString).exists()) { throw new DbImportException("File does not exist for column '" + columnName + "': " + valueString); + } else if (maxBlobSize >= 0 && maxBlobSize < new File(valueString).length()) { + if (dbDefinition.getDbVendor() == DbVendor.MySQL) { + throw new Exception("File size is too big for current database settings. Please adjust MySQL server variable 'max_allowed_packet' to at least " + new File(valueString).length()); + } else if (dbDefinition.getDbVendor() == DbVendor.MariaDB) { + throw new Exception("File size is too big for current database settings. Please adjust MariaDB server variable 'max_allowed_packet' to at least " + new File(valueString).length()); + } } else if (simpleDataType == SimpleDataType.Blob) { InputStream inputStream; if (Utilities.endsWithIgnoreCase(valueString, ".zip")) { diff --git a/src/de/soderer/dbimport/DbNoSqlImportWorker.java b/src/de/soderer/dbimport/DbNoSqlImportWorker.java index e7e9501..56443d0 100644 --- a/src/de/soderer/dbimport/DbNoSqlImportWorker.java +++ b/src/de/soderer/dbimport/DbNoSqlImportWorker.java @@ -262,7 +262,7 @@ protected void insertItemsWithoutKeyColumns(final Connection connection, final M final String formatInfo = mappingToUse.get(unescapedDbColumnToInsert).getSecond(); final - Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry); + Closeable itemToClose = validateAndSetParameter(preparedInsertStatement, i++, dbColumnToInsert, simpleDataType, dbColumns.get(dbColumnToInsert).isNullable(), dataValue, formatInfo, batchValueEntry, -1); if (itemToClose != null) { itemsToCloseAfterwards.add(itemToClose); } @@ -414,7 +414,7 @@ private void updateAndInsertItems(final Connection connection, final Map 0) { @@ -926,6 +928,30 @@ public void testCsvImportInsertBlobFile() { } } + @Test + public void testCsvImportInsertBigBlobFile() { + try { + createEmptyTestTable(); + prefillTestTable(); + + final byte[] randomByteArray = new byte[20 * 1024 * 1024]; + new Random().nextBytes(randomByteArray); + FileUtilities.write(BLOB_DATA_FILE, randomByteArray); + + Assert.assertEquals(0, DbImport._main(new String[] { + "importblob", + "mariadb", + HOSTNAME, + DBNAME, + USERNAME, + "-updatesql", "UPDATE test_tbl SET column_blob = ? WHERE column_integer = 1", + "-blobfile", BLOB_DATA_FILE.getAbsolutePath(), + PASSWORD })); + } catch (final Exception e) { + Assert.fail(e.getMessage()); + } + } + @Test public void testCsvImportCreateTable() { try {