Skip to content

Commit

Permalink
initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
sunchao committed Sep 23, 2024
1 parent 437db30 commit 50b7efa
Showing 1 changed file with 25 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.sql.delta.commands
import java.util.concurrent.TimeUnit.NANOSECONDS

import scala.util.control.NonFatal
import scala.collection.mutable.{Map => MutableMap, Set => MutableSet}

import org.apache.spark.sql.delta.{DeltaAnalysisException, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableIdentifier, DeltaTableUtils, OptimisticTransaction, ResolvedPathBasedNonDeltaTable}
import org.apache.spark.sql.delta.actions._
Expand Down Expand Up @@ -100,11 +101,30 @@ trait DeltaCommand extends DeltaLogging {
def generateCandidateFileMap(
basePath: Path,
candidateFiles: Seq[AddFile]): Map[String, AddFile] = {
val nameToAddFileMap = candidateFiles.map(add =>
DeltaFileOperations.absolutePath(basePath.toString, add.path).toString -> add).toMap
assert(nameToAddFileMap.size == candidateFiles.length,
s"File name collisions found among:\n${candidateFiles.map(_.path).mkString("\n")}")
nameToAddFileMap
val nameToAddFileMap = MutableMap[String, AddFile]()
val duplicatePaths = MutableMap[String, MutableSet[AddFile]]()

candidateFiles.foreach { add: AddFile =>
val filePath = DeltaFileOperations.absolutePath(basePath.toString, add.path).toString
if (nameToAddFileMap.contains(filePath)) {
// Add the file to the set of conflicts for this path
duplicatePaths.getOrElseUpdate(filePath, MutableSet(nameToAddFileMap(filePath))).add(add)
} else {
nameToAddFileMap += (filePath -> add)
}
}

// Check if there were any duplicates and throw an exception if found
if (duplicatePaths.nonEmpty) {
throw new IllegalArgumentException(
s"File name collisions detected for the following paths:\n" +
duplicatePaths.map { case (filePath, files) =>
s"Path: $filePath\nConflicting files: ${files.map(_.path).mkString(", ")}"
}.mkString("\n\n")
)
}

nameToAddFileMap.toMap
}

/**
Expand Down

0 comments on commit 50b7efa

Please sign in to comment.