From de7b86a5dd24d9cff4fa28ca7b8eadbd50ed8bb9 Mon Sep 17 00:00:00 2001 From: ZhilinLi Date: Tue, 12 Sep 2023 15:21:05 +0800 Subject: [PATCH] [fix][hive-source][bug] fix An error occurred reading an empty directory (#5427) * [fix][hive-source][bug] fix An error occurred reading an empty directory * [fix][hive-source][bug] fix An error occurred reading an empty directory --- .../source/reader/AbstractReadStrategy.java | 22 +++++++------------ .../file/source/reader/ExcelReadStrategy.java | 3 ++- .../file/source/reader/TextReadStrategy.java | 3 ++- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java index e4e1694f30d..1dc8e7a2ad9 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/AbstractReadStrategy.java @@ -24,8 +24,6 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSourceConfig; import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf; -import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException; import org.apache.seatunnel.connectors.seatunnel.file.sink.util.FileSystemUtils; import org.apache.hadoop.conf.Configuration; @@ -153,15 +151,9 @@ public List getFileNamesByPath(HadoopConf hadoopConf, String path) throw } } } - - if (fileNames.isEmpty()) { - throw new FileConnectorException( - FileConnectorErrorCode.FILE_LIST_EMPTY, - "The target file list is empty," - + "SeaTunnel will not be able to sync empty table, " - + "please check the configuration parameters such as: [file_filter_pattern]"); + if (this.fileNames.isEmpty()) { + log.error("The current directory is empty " + path); } - return fileNames; } @@ -196,10 +188,12 @@ public SeaTunnelRowType getActualSeaTunnelRowTypeInfo() { protected Map parsePartitionsByPath(String path) { LinkedHashMap partitions = new LinkedHashMap<>(); - Arrays.stream(path.split("/", -1)) - .filter(split -> split.contains("=")) - .map(split -> split.split("=", -1)) - .forEach(kv -> partitions.put(kv[0], kv[1])); + if (path != null && !path.isEmpty()) { + Arrays.stream(path.split("/", -1)) + .filter(split -> split.contains("=")) + .map(split -> split.split("=", -1)) + .forEach(kv -> partitions.put(kv[0], kv[1])); + } return partitions; } diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java index 0b1cfc083ba..649216c752c 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/ExcelReadStrategy.java @@ -136,7 +136,8 @@ public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) { "Schmea information is not set or incorrect schmea settings"); } SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes(fileNames.get(0), seaTunnelRowType); + mergePartitionTypes( + fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); // column projection if (pluginConfig.hasPath(BaseSourceConfig.READ_COLUMNS.key())) { // get the read column index from user-defined row type diff --git a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java index 4b931cb8902..eefa8e4de76 100644 --- a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java +++ b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/TextReadStrategy.java @@ -138,7 +138,8 @@ public SeaTunnelRowType getSeaTunnelRowTypeInfo(HadoopConf hadoopConf, String pa @Override public void setSeaTunnelRowTypeInfo(SeaTunnelRowType seaTunnelRowType) { SeaTunnelRowType userDefinedRowTypeWithPartition = - mergePartitionTypes(fileNames.get(0), seaTunnelRowType); + mergePartitionTypes( + fileNames.size() > 0 ? fileNames.get(0) : null, seaTunnelRowType); if (pluginConfig.hasPath(BaseSourceConfig.DELIMITER.key())) { fieldDelimiter = pluginConfig.getString(BaseSourceConfig.DELIMITER.key()); } else {