Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
piffall committed Oct 3, 2023
2 parents d001c47 + 53b94cd commit fdc34a7
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ object DatePartitionFormatter {
apply(cols)
}

def apply(definitions: Seq[DatePartColumn]): DatePartitionFormatter = {
def apply(definitions: Seq[DatePartColumn], hasLabels: Boolean = true): DatePartitionFormatter = {
if (definitions.isEmpty)
throw new IllegalArgumentException(
"Column definitions for a DatePartitionFormatter cannot be empty")
Expand All @@ -60,18 +60,23 @@ object DatePartitionFormatter {
throw new IllegalArgumentException(msg, ex)
}
}
new DatePartitionFormatter(cols)
new DatePartitionFormatter(cols, hasLabels)
}

}

class DatePartitionFormatter protected (
columns: Seq[DatePartitionFormatter.ColumnFormatter]) {
columns: Seq[DatePartitionFormatter.ColumnFormatter], hasLabels: Boolean = true) {

def dateToPath(date: LocalDateTime): String = {
columns
.map { part =>
s"${part.column}=${part.formatter.format(date)}"
if (hasLabels) {
s"${part.column}=${part.formatter.format(date)}"
} else {
part.formatter.format(date)
}

}
.mkString("/")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,5 +132,26 @@ class DatePartitionsTest extends WordSpec with MockFactory {
.generatePaths(from, to)
}
}

"partitioning only contains date, but not labels for column" should {
"return a proper list" in {
val from = LocalDateTime.of(2020, 5, 12, 0, 0)
val to = LocalDateTime.of(2020, 5, 15, 0, 0)

val fsStub = stub[FileSystem]
(fsStub.pathExists _).when("2020-05-12").returns(true)
(fsStub.pathExists _).when("2020-05-13").returns(true)
(fsStub.pathExists _).when("2020-05-14").returns(true)
(fsStub.pathExists _).when("2020-05-15").returns(true)

val expected = "2020-05-12" :: "2020-05-13" :: "2020-05-14" :: "2020-05-15" :: Nil

val generated = DatePartitions(
fsStub, DatePartitionFormatter(DatePartColumn("dt", "yyyy-MM-dd") :: Nil, hasLabels = false))
.generatePaths(from, to)

assert(expected === generated)
}
}
}
}

0 comments on commit fdc34a7

Please sign in to comment.