diff --git a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/partitioning/DatePartitionFormatter.scala b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/partitioning/DatePartitionFormatter.scala index 667a9b5..3d29c2c 100644 --- a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/partitioning/DatePartitionFormatter.scala +++ b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/partitioning/DatePartitionFormatter.scala @@ -43,7 +43,7 @@ object DatePartitionFormatter { apply(cols) } - def apply(definitions: Seq[DatePartColumn]): DatePartitionFormatter = { + def apply(definitions: Seq[DatePartColumn], hasLabels: Boolean = true): DatePartitionFormatter = { if (definitions.isEmpty) throw new IllegalArgumentException( "Column definitions for a DatePartitionFormatter cannot be empty") @@ -60,18 +60,23 @@ object DatePartitionFormatter { throw new IllegalArgumentException(msg, ex) } } - new DatePartitionFormatter(cols) + new DatePartitionFormatter(cols, hasLabels) } } class DatePartitionFormatter protected ( - columns: Seq[DatePartitionFormatter.ColumnFormatter]) { + columns: Seq[DatePartitionFormatter.ColumnFormatter], hasLabels: Boolean = true) { def dateToPath(date: LocalDateTime): String = { columns .map { part => - s"${part.column}=${part.formatter.format(date)}" + if (hasLabels) { + s"${part.column}=${part.formatter.format(date)}" + } else { + part.formatter.format(date) + } + } .mkString("/") } diff --git a/damavis-spark-core/src/test/scala/com/damavis/spark/resource/partitioning/DatePartitionsTest.scala b/damavis-spark-core/src/test/scala/com/damavis/spark/resource/partitioning/DatePartitionsTest.scala index dbd6907..89ec0d5 100644 --- a/damavis-spark-core/src/test/scala/com/damavis/spark/resource/partitioning/DatePartitionsTest.scala +++ b/damavis-spark-core/src/test/scala/com/damavis/spark/resource/partitioning/DatePartitionsTest.scala @@ -132,5 +132,26 @@ class DatePartitionsTest extends WordSpec with MockFactory { .generatePaths(from, to) } } + + "partitioning only contains date, but not labels for column" should { + "return a proper list" in { + val from = LocalDateTime.of(2020, 5, 12, 0, 0) + val to = LocalDateTime.of(2020, 5, 15, 0, 0) + + val fsStub = stub[FileSystem] + (fsStub.pathExists _).when("2020-05-12").returns(true) + (fsStub.pathExists _).when("2020-05-13").returns(true) + (fsStub.pathExists _).when("2020-05-14").returns(true) + (fsStub.pathExists _).when("2020-05-15").returns(true) + + val expected = "2020-05-12" :: "2020-05-13" :: "2020-05-14" :: "2020-05-15" :: Nil + + val generated = DatePartitions( + fsStub, DatePartitionFormatter(DatePartColumn("dt", "yyyy-MM-dd") :: Nil, hasLabels = false)) + .generatePaths(from, to) + + assert(expected === generated) + } + } } }