From 431d6a3650f332684d078a67269aa01d315c47d5 Mon Sep 17 00:00:00 2001 From: Paul Sasieta Date: Fri, 3 Nov 2023 08:37:15 +0100 Subject: [PATCH] Options not overwritten --- .../com/damavis/spark/resource/file/FileReaderBuilder.scala | 2 +- .../com/damavis/spark/resource/file/FileWriterBuilder.scala | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileReaderBuilder.scala b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileReaderBuilder.scala index 20539f2..f42c6d8 100644 --- a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileReaderBuilder.scala +++ b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileReaderBuilder.scala @@ -38,7 +38,7 @@ class FileReaderBuilder(params: FileReaderParameters)(implicit spark: SparkSessi } def options(options: Map[String, String]): FileReaderBuilder = { - val newParams = params.copy(options = options) + val newParams = params.copy(options = params.options ++ options) new FileReaderBuilder(newParams) } diff --git a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileWriterBuilder.scala b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileWriterBuilder.scala index 17d4111..207a927 100644 --- a/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileWriterBuilder.scala +++ b/damavis-spark-core/src/main/scala/com/damavis/spark/resource/file/FileWriterBuilder.scala @@ -21,8 +21,10 @@ class FileWriterBuilder(params: FileWriterParameters) extends WriterBuilder { def mode(mode: String): FileWriterBuilder = new FileWriterBuilder(params.copy(mode = mode)) - def options(opts: Map[String, String]): FileWriterBuilder = - new FileWriterBuilder(params.copy(options = opts)) + def options(opts: Map[String, String]): FileWriterBuilder = { + val newParams = params.copy(options = params.options ++ opts) + new FileWriterBuilder(newParams) + } def partitionedBy(columnNames: Seq[String]): FileWriterBuilder = { if (columnNames.isEmpty)