From c6c3f8cd2b0b5b0b03a3e046671450809ea013af Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 24 May 2018 20:08:28 +0100
Subject: [PATCH 01/25] Create implicit class to attach saveAsSingleTextFile to
 RDDs

---
 .../scala/com/spark_helper/SparkHelper.scala  | 79 +++++++++++++++++++
 .../com/spark_helper/SparkHelperTest.scala    | 77 +++++++++++-------
 2 files changed, 126 insertions(+), 30 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 3f612eb..dcc7a8e 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -42,6 +42,85 @@ import scala.util.Random
   */
 object SparkHelper extends Serializable {
 
+  implicit class RDDExtensions(val rdd: RDD[String]) extends AnyVal {
+
+    /** Saves an RDD in exactly one file.
+      *
+      * Allows one to save an RDD in one file, while keeping the processing
+      * parallelized.
+      *
+      * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt") }}}
+      *
+      * @param outputFile the path of the produced file
+      */
+    def saveAsSingleTextFile(outputFile: String): Unit =
+      SparkHelper.saveAsSingleTextFile(rdd, outputFile)
+
+    /** Saves an RDD in exactly one file.
+      *
+      * Allows one to save an RDD in one file, while keeping the processing
+      * parallelized.
+      *
+      * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", classOf[BZip2Codec]) }}}
+      *
+      * @param outputFile the path of the produced file
+      * @param compressionCodec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def saveAsSingleTextFile(
+        outputFile: String,
+        compressionCodec: Class[_ <: CompressionCodec]
+    ): Unit =
+      SparkHelper.saveAsSingleTextFile(rdd, outputFile, compressionCodec)
+
+    /** Saves an RDD in exactly one file.
+      *
+      * Allows one to save an RDD in one file, while keeping the processing
+      * parallelized.
+      *
+      * This variant of saveAsSingleTextFile performs the storage in a temporary
+      * folder instead of directly in the final output folder. This way the
+      * risks of having corrupted files in the real output folder due to cluster
+      * interruptions is minimized.
+      *
+      * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path") }}}
+      *
+      * @param outputFile the path of the produced file
+      * @param workingFolder the path where file manipulations will temporarily
+      * happen.
+      */
+    def saveAsSingleTextFile(outputFile: String, workingFolder: String): Unit =
+      SparkHelper.saveAsSingleTextFile(rdd, outputFile, workingFolder)
+
+    /** Saves an RDD in exactly one file.
+      *
+      * Allows one to save an RDD in one file, while keeping the processing
+      * parallelized.
+      *
+      * This variant of saveAsSingleTextFile performs the storage in a temporary
+      * folder instead of directly in the final output folder. This way the risks
+      * of having corrupted files in the real output folder due to cluster
+      * interruptions is minimized.
+      *
+      * {{{
+      * rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path", classOf[BZip2Codec])
+      * }}}
+      *
+      * @param outputFile the path of the produced file
+      * @param workingFolder the path where file manipulations will temporarily
+      * happen.
+      * @param compressionCodec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def saveAsSingleTextFile(
+        outputFile: String,
+        workingFolder: String,
+        compressionCodec: Class[_ <: CompressionCodec]
+    ): Unit =
+      SparkHelper
+        .saveAsSingleTextFile(rdd, outputFile, workingFolder, compressionCodec)
+  }
+
   /** Saves an RDD in exactly one file.
     *
     * Allows one to save an RDD in one file, while keeping the processing
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index b3578df..3e60a7a 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,5 +1,9 @@
 package com.spark_helper
 
+import com.spark_helper.SparkHelper.RDDExtensions
+
+import org.apache.hadoop.io.compress.GzipCodec
+
 import com.holdenkarau.spark.testing.{SharedSparkContext, RDDComparisons}
 
 import org.scalatest.FunSuite
@@ -14,54 +18,67 @@ class SparkHelperTest
     with SharedSparkContext
     with RDDComparisons {
 
+  val resourceFolder = "src/test/resources"
+
   test("Save as single text file") {
 
+    val testFolder = s"$resourceFolder/folder"
+    val singleTextFilePath = s"$testFolder/single_text_file.txt"
+    val tmpFolder = s"$resourceFolder/tmp"
+
+    HdfsHelper.deleteFolder(testFolder)
+    HdfsHelper.deleteFolder(tmpFolder)
+
+    val rddToStore =
+      sc.parallelize(Array("data_a", "data_b", "data_c")).repartition(3)
+
     // 1: Without an intermediate working dir:
 
-    var repartitionedDataToStore = sc
-      .parallelize(Array("data_a", "data_b", "data_c"))
-      .repartition(3)
+    SparkHelper.saveAsSingleTextFile(rddToStore, singleTextFilePath)
 
-    HdfsHelper.deleteFile("src/test/resources/single_text_file.txt")
-    SparkHelper.saveAsSingleTextFile(
-      repartitionedDataToStore,
-      "src/test/resources/single_text_file.txt")
+    var singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
 
-    var singleFileStoredData = sc
-      .textFile("src/test/resources/single_text_file.txt")
-      .collect()
-      .sorted
+    assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
+
+    HdfsHelper.deleteFolder(testFolder)
+
+    // 1-bis: same, but using the implicit RDD extension:
+
+    rddToStore.saveAsSingleTextFile(singleTextFilePath)
+
+    singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
 
     assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
 
-    HdfsHelper.deleteFile("src/test/resources/single_text_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
 
     // 2: With an intermediate working dir:
     // Notice as well that we test by moving the single file in a folder
     // which doesn't exists.
 
-    repartitionedDataToStore = sc
-      .parallelize(Array("data_a", "data_b", "data_c"))
-      .repartition(3)
-
-    HdfsHelper.deleteFile("src/test/resources/folder/single_text_file.txt")
-    HdfsHelper.deleteFolder("src/test/resources/folder")
     SparkHelper.saveAsSingleTextFile(
-      repartitionedDataToStore,
-      "src/test/resources/folder/single_text_file.txt",
-      workingFolder = "src/test/resources/tmp")
-    assert(
-      HdfsHelper.fileExists("src/test/resources/folder/single_text_file.txt"))
-
-    singleFileStoredData = sc
-      .textFile("src/test/resources/folder/single_text_file.txt")
-      .collect()
-      .sorted
+      rddToStore,
+      singleTextFilePath,
+      workingFolder = tmpFolder)
+
+    singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
+
+    assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
+
+    HdfsHelper.deleteFolder(testFolder)
+    HdfsHelper.deleteFolder(tmpFolder)
+
+    // 3: With a compression codec:
+
+    rddToStore
+      .saveAsSingleTextFile(s"$singleTextFilePath.gz", classOf[GzipCodec])
+
+    singleFileStoredData =
+      sc.textFile(s"$singleTextFilePath.gz").collect().sorted
 
     assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
 
-    HdfsHelper.deleteFolder("src/test/resources/folder")
-    HdfsHelper.deleteFolder("src/test/resources/tmp")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Read text file with specific record delimiter") {

From a23924667af312246d1b474d942aa730e4fe71be Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 24 May 2018 20:18:00 +0100
Subject: [PATCH 02/25] Remove previous saveAsSingleTextFile by the ones
 implicitely attached to the SparkContext

---
 .../scala/com/spark_helper/SparkHelper.scala  | 123 +++---------------
 .../com/spark_helper/SparkHelperTest.scala    |  39 +++---
 2 files changed, 31 insertions(+), 131 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index dcc7a8e..03d2764 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -54,7 +54,7 @@ object SparkHelper extends Serializable {
       * @param outputFile the path of the produced file
       */
     def saveAsSingleTextFile(outputFile: String): Unit =
-      SparkHelper.saveAsSingleTextFile(rdd, outputFile)
+      SparkHelper.saveAsSingleTextFileInternal(rdd, outputFile, None)
 
     /** Saves an RDD in exactly one file.
       *
@@ -71,7 +71,8 @@ object SparkHelper extends Serializable {
         outputFile: String,
         compressionCodec: Class[_ <: CompressionCodec]
     ): Unit =
-      SparkHelper.saveAsSingleTextFile(rdd, outputFile, compressionCodec)
+      SparkHelper
+        .saveAsSingleTextFileInternal(rdd, outputFile, Some(compressionCodec))
 
     /** Saves an RDD in exactly one file.
       *
@@ -90,7 +91,12 @@ object SparkHelper extends Serializable {
       * happen.
       */
     def saveAsSingleTextFile(outputFile: String, workingFolder: String): Unit =
-      SparkHelper.saveAsSingleTextFile(rdd, outputFile, workingFolder)
+      SparkHelper.saveAsSingleTextFileWithWorkingFolderInternal(
+        rdd,
+        outputFile,
+        workingFolder,
+        None
+      )
 
     /** Saves an RDD in exactly one file.
       *
@@ -117,113 +123,14 @@ object SparkHelper extends Serializable {
         workingFolder: String,
         compressionCodec: Class[_ <: CompressionCodec]
     ): Unit =
-      SparkHelper
-        .saveAsSingleTextFile(rdd, outputFile, workingFolder, compressionCodec)
+      SparkHelper.saveAsSingleTextFileWithWorkingFolderInternal(
+        rdd,
+        outputFile,
+        workingFolder,
+        Some(compressionCodec)
+      )
   }
 
-  /** Saves an RDD in exactly one file.
-    *
-    * Allows one to save an RDD in one file, while keeping the processing
-    * parallelized.
-    *
-    * {{{ SparkHelper.saveAsSingleTextFile(myRddToStore, "/my/file/path.txt") }}}
-    *
-    * @param outputRDD the RDD of strings to store in one file
-    * @param outputFile the path of the produced file
-    */
-  def saveAsSingleTextFile(outputRDD: RDD[String], outputFile: String): Unit =
-    saveAsSingleTextFileInternal(outputRDD, outputFile, None)
-
-  /** Saves an RDD in exactly one file.
-    *
-    * Allows one to save an RDD in one file, while keeping the processing
-    * parallelized.
-    *
-    * {{{
-    * SparkHelper.saveAsSingleTextFile(
-    *   myRddToStore, "/my/file/path.txt", classOf[BZip2Codec])
-    * }}}
-    *
-    * @param outputRDD the RDD of strings to store in one file
-    * @param outputFile the path of the produced file
-    * @param compressionCodec the type of compression to use (for instance
-    * classOf[BZip2Codec] or classOf[GzipCodec]))
-    */
-  def saveAsSingleTextFile(
-      outputRDD: RDD[String],
-      outputFile: String,
-      compressionCodec: Class[_ <: CompressionCodec]
-  ): Unit =
-    saveAsSingleTextFileInternal(outputRDD, outputFile, Some(compressionCodec))
-
-  /** Saves an RDD in exactly one file.
-    *
-    * Allows one to save an RDD in one file, while keeping the processing
-    * parallelized.
-    *
-    * This variant of saveAsSingleTextFile performs the storage in a temporary
-    * folder instead of directly in the final output folder. This way the
-    * risks of having corrupted files in the real output folder due to cluster
-    * interruptions is minimized.
-    *
-    * {{{
-    * SparkHelper.saveAsSingleTextFile(
-    *   myRddToStore, "/my/file/path.txt", "/my/working/folder/path")
-    * }}}
-    *
-    * @param outputRDD the RDD of strings to store in one file
-    * @param outputFile the path of the produced file
-    * @param workingFolder the path where file manipulations will temporarily
-    * happen.
-    */
-  def saveAsSingleTextFile(
-      outputRDD: RDD[String],
-      outputFile: String,
-      workingFolder: String
-  ): Unit =
-    saveAsSingleTextFileWithWorkingFolderInternal(
-      outputRDD,
-      outputFile,
-      workingFolder,
-      None)
-
-  /** Saves an RDD in exactly one file.
-    *
-    * Allows one to save an RDD in one file, while keeping the processing
-    * parallelized.
-    *
-    * This variant of saveAsSingleTextFile performs the storage in a temporary
-    * folder instead of directly in the final output folder. This way the risks
-    * of having corrupted files in the real output folder due to cluster
-    * interruptions is minimized.
-    *
-    * {{{
-    * SparkHelper.saveAsSingleTextFile(
-    *   myRddToStore,
-    *   "/my/file/path.txt",
-    *   "/my/working/folder/path",
-    *   classOf[BZip2Codec])
-    * }}}
-    *
-    * @param outputRDD the RDD of strings to store in one file
-    * @param outputFile the path of the produced file
-    * @param workingFolder the path where file manipulations will temporarily
-    * happen.
-    * @param compressionCodec the type of compression to use (for instance
-    * classOf[BZip2Codec] or classOf[GzipCodec]))
-    */
-  def saveAsSingleTextFile(
-      outputRDD: RDD[String],
-      outputFile: String,
-      workingFolder: String,
-      compressionCodec: Class[_ <: CompressionCodec]
-  ): Unit =
-    saveAsSingleTextFileWithWorkingFolderInternal(
-      outputRDD,
-      outputFile,
-      workingFolder,
-      Some(compressionCodec))
-
   /** Equivalent to sparkContext.textFile(), but for a specific record delimiter.
     *
     * By default, sparkContext.textFile() will provide one record per line. But
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 3e60a7a..4272a73 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -34,19 +34,9 @@ class SparkHelperTest
 
     // 1: Without an intermediate working dir:
 
-    SparkHelper.saveAsSingleTextFile(rddToStore, singleTextFilePath)
-
-    var singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
-
-    assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
-
-    HdfsHelper.deleteFolder(testFolder)
-
-    // 1-bis: same, but using the implicit RDD extension:
-
     rddToStore.saveAsSingleTextFile(singleTextFilePath)
 
-    singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
+    var singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
 
     assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
 
@@ -56,10 +46,10 @@ class SparkHelperTest
     // Notice as well that we test by moving the single file in a folder
     // which doesn't exists.
 
-    SparkHelper.saveAsSingleTextFile(
-      rddToStore,
+    rddToStore.saveAsSingleTextFile(
       singleTextFilePath,
-      workingFolder = tmpFolder)
+      workingFolder = tmpFolder
+    )
 
     singleFileStoredData = sc.textFile(singleTextFilePath).collect().sorted
 
@@ -229,15 +219,18 @@ class SparkHelperTest
     HdfsHelper.deleteFolder("src/test/resources/re_coalescence_test_output")
 
     // Let's create the folder with high level of coalescence (3 files):
-    SparkHelper.saveAsSingleTextFile(
-      sc.parallelize[String](Array("data_1_a", "data_1_b", "data_1_c")),
-      "src/test/resources/re_coalescence_test_input/input_file_1")
-    SparkHelper.saveAsSingleTextFile(
-      sc.parallelize[String](Array("data_2_a", "data_2_b")),
-      "src/test/resources/re_coalescence_test_input/input_file_2")
-    SparkHelper.saveAsSingleTextFile(
-      sc.parallelize[String](Array("data_3_a", "data_3_b", "data_3_c")),
-      "src/test/resources/re_coalescence_test_input/input_file_3")
+    sc.parallelize[String](Array("data_1_a", "data_1_b", "data_1_c"))
+      .saveAsSingleTextFile(
+        "src/test/resources/re_coalescence_test_input/input_file_1"
+      )
+    sc.parallelize[String](Array("data_2_a", "data_2_b"))
+      .saveAsSingleTextFile(
+        "src/test/resources/re_coalescence_test_input/input_file_2"
+      )
+    sc.parallelize[String](Array("data_3_a", "data_3_b", "data_3_c"))
+      .saveAsSingleTextFile(
+        "src/test/resources/re_coalescence_test_input/input_file_3"
+      )
 
     // Let's decrease the coalescence level in order to only have 2 files:
     SparkHelper.decreaseCoalescence(

From 592564f69b084b20ce3f333b9188a0b7fd7ac6fb Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 24 May 2018 20:32:16 +0100
Subject: [PATCH 03/25] Clean-up saveAsSingleTextFile internals

---
 .../scala/com/spark_helper/SparkHelper.scala    | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 03d2764..b30400e 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -545,7 +545,7 @@ object SparkHelper extends Serializable {
 
     // We chose a random name for the temporary file:
     val temporaryName = Random.alphanumeric.take(10).mkString("")
-    val temporaryFile = workingFolder + "/" + temporaryName
+    val temporaryFile = s"$workingFolder/$temporaryName"
 
     // We perform the merge into a temporary single text file:
     saveAsSingleTextFileInternal(outputRDD, temporaryFile, compressionCodec)
@@ -569,28 +569,29 @@ object SparkHelper extends Serializable {
       compressionCodec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
+    val hadoopConfiguration = outputRDD.sparkContext.hadoopConfiguration
+    val fileSystem = FileSystem.get(hadoopConfiguration)
 
     // Classic saveAsTextFile in a temporary folder:
-    HdfsHelper.deleteFolder(outputFile + ".tmp")
+    HdfsHelper.deleteFolder(s"$outputFile.tmp")
     compressionCodec match {
       case Some(compressionCodec) =>
-        outputRDD.saveAsTextFile(outputFile + ".tmp", compressionCodec)
+        outputRDD.saveAsTextFile(s"$outputFile.tmp", compressionCodec)
       case None =>
-        outputRDD.saveAsTextFile(outputFile + ".tmp")
+        outputRDD.saveAsTextFile(s"$outputFile.tmp")
     }
 
     // Merge the folder into a single file:
     HdfsHelper.deleteFile(outputFile)
     FileUtil.copyMerge(
       fileSystem,
-      new Path(outputFile + ".tmp"),
+      new Path(s"$outputFile.tmp"),
       fileSystem,
       new Path(outputFile),
       true,
-      new Configuration(),
+      hadoopConfiguration,
       null)
-    HdfsHelper.deleteFolder(outputFile + ".tmp")
+    HdfsHelper.deleteFolder(s"$outputFile.tmp")
   }
 
   private def decreaseCoalescenceInternal(

From 19b88c35671037b45bc0bc48d7eada7351fd7358 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 24 May 2018 20:56:50 +0100
Subject: [PATCH 04/25] Create implicit class to attach an additional textFile
 to SparkContext which splits record on a specific delimiter

---
 .../scala/com/spark_helper/SparkHelper.scala  | 153 +++++++++---------
 .../com/spark_helper/SparkHelperTest.scala    |  39 ++---
 2 files changed, 90 insertions(+), 102 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index b30400e..12e0164 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -131,84 +131,83 @@ object SparkHelper extends Serializable {
       )
   }
 
-  /** Equivalent to sparkContext.textFile(), but for a specific record delimiter.
-    *
-    * By default, sparkContext.textFile() will provide one record per line. But
-    * what if the format you want to read considers that one record (one entity)
-    * is stored in more than one line (yml, xml, ...)?
-    *
-    * For instance in order to read a yml file, which is a format for which a
-    * record (a single entity) is spread other several lines, you can modify the
-    * record delimiter with "---\n" instead of "\n". Same goes when reading an
-    * xml file where a record might be spread over several lines or worse the
-    * whole xml file is one line.
-    *
-    * {{{
-    * // Let's say data we want to use with Spark looks like this (one record is
-    * // a customer, but it's spread over several lines):
-    * <Customers>\n
-    * <Customer>\n
-    * <Address>34 thingy street, someplace, sometown</Address>\n
-    * </Customer>\n
-    * <Customer>\n
-    * <Address>12 thingy street, someplace, sometown</Address>\n
-    * </Customer>\n
-    * </Customers>
-    * //Then you can use it this way:
-    * val computedRecords = SparkHelper.textFileWithDelimiter(
-    *   "my/path/to/customers.xml", sparkContext, <Customer>\n
-    * ).collect()
-    * val expectedRecords = Array(
-    *   <Customers>\n,
-    *   (
-    *     <Address>34 thingy street, someplace, sometown</Address>\n +
-    *     </Customer>\n
-    *   ),
-    *   (
-    *     <Address>12 thingy street, someplace, sometown</Address>\n +
-    *     </Customer>\n +
-    *     </Customers>
-    *   )
-    * )
-    * assert(computedRecords == expectedRecords)
-    * }}}
-    *
-    * @param hdfsPath the path of the file to read (folder or file, '*' works as
-    * well).
-    * @param sparkContext the SparkContext
-    * @param delimiter the specific record delimiter which replaces "\n"
-    * @param maxRecordLength the max length (not sure which unit) of a record
-    * before considering the record too long to fit into memory.
-    * @return the RDD of records
-    */
-  def textFileWithDelimiter(
-      hdfsPath: String,
-      sparkContext: SparkContext,
-      delimiter: String,
-      maxRecordLength: String = "1000000"
-  ): RDD[String] = {
-
-    val conf = new Configuration(sparkContext.hadoopConfiguration)
-
-    // This configuration sets the record delimiter:
-    conf.set("textinputformat.record.delimiter", delimiter)
-
-    // and this one limits the size of one record. This is necessary in order to
-    // avoid reading from a corrupted file from which a record could be too long
-    // to fit in memory. This way, when reading a corrupted file, this will
-    // throw an exception (java.io.IOException - thus catchable) rather than
-    // having a messy out of memory which will stop the sparkContext:
-    conf.set("mapreduce.input.linerecordreader.line.maxlength", maxRecordLength)
+  implicit class SparkContextExtensions(val sc: SparkContext) extends AnyVal {
 
-    sparkContext
-      .newAPIHadoopFile(
-        hdfsPath,
-        classOf[TextInputFormat],
-        classOf[LongWritable],
-        classOf[Text],
-        conf
-      )
-      .map { case (_, text) => text.toString }
+    /** Equivalent to sparkContext.textFile(), but for a specific record delimiter.
+      *
+      * By default, sparkContext.textFile() will provide one record per line
+      * (per '\n'). But what if the format to read considers that one record
+      * is stored in more than one line (yml, custom format, ...)?
+      *
+      * For instance in order to read a yml file, which is a format for which a
+      * record (a single entity) is spread other several lines, you can modify
+      * the record delimiter with "---\n" instead of "\n". Same goes when
+      * reading an xml file where a record might be spread over several lines or
+      * worse the whole xml file is one line.
+      *
+      * {{{
+      * // Let's say data we want to use with Spark looks like this (one record
+      * // is a customer, but it's spread over several lines):
+      * <Customers>\n
+      * <Customer>\n
+      * <Address>34 thingy street, someplace, sometown</Address>\n
+      * </Customer>\n
+      * <Customer>\n
+      * <Address>12 thingy street, someplace, sometown</Address>\n
+      * </Customer>\n
+      * </Customers>
+      * //Then you can use it this way:
+      * val computedRecords = sc.textFile("my/path/to/customers.xml", "<Customer>\n")
+      * val expectedRecords = RDD(
+      *   <Customers>\n,
+      *   (
+      *     <Address>34 thingy street, someplace, sometown</Address>\n +
+      *     </Customer>\n
+      *   ),
+      *   (
+      *     <Address>12 thingy street, someplace, sometown</Address>\n +
+      *     </Customer>\n +
+      *     </Customers>
+      *   )
+      * )
+      * assert(computedRecords == expectedRecords)
+      * }}}
+      *
+      * @param hdfsPath the path of the file to read (folder or file, '*' works
+      * as well).
+      * @param delimiter the specific record delimiter which replaces "\n"
+      * @param maxRecordLength the max length (not sure which unit) of a record
+      * before considering the record too long to fit into memory.
+      * @return the RDD of records
+      */
+    def textFile(
+        hdfsPath: String,
+        delimiter: String,
+        maxRecordLength: String = "1000000"
+    ): RDD[String] = {
+
+      val conf = new Configuration(sc.hadoopConfiguration)
+
+      // This configuration sets the record delimiter:
+      conf.set("textinputformat.record.delimiter", delimiter)
+
+      // and this one limits the size of one record. This is necessary in order
+      // to avoid reading from a corrupted file from which a record could be too
+      // long to fit in memory. This way, when reading a corrupted file, this
+      // will throw an exception (java.io.IOException - thus catchable) rather
+      // than having a messy out of memory which will stop the sparkContext:
+      conf
+        .set("mapreduce.input.linerecordreader.line.maxlength", maxRecordLength)
+
+      sc.newAPIHadoopFile(
+          hdfsPath,
+          classOf[TextInputFormat],
+          classOf[LongWritable],
+          classOf[Text],
+          conf
+        )
+        .map { case (_, text) => text.toString }
+    }
   }
 
   /** Saves and repartitions a key/value RDD on files whose name is the key.
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 4272a73..c476928 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,6 +1,6 @@
 package com.spark_helper
 
-import com.spark_helper.SparkHelper.RDDExtensions
+import com.spark_helper.SparkHelper.{RDDExtensions, SparkContextExtensions}
 
 import org.apache.hadoop.io.compress.GzipCodec
 
@@ -73,10 +73,12 @@ class SparkHelperTest
 
   test("Read text file with specific record delimiter") {
 
+    val weirdFormatFilePath = s"$resourceFolder/some_weird_format.txt"
+
     // 1: Let's read a file where a record begins with a line begining with
     // 3 and other lines begining by 4:
 
-    HdfsHelper.deleteFile("src/test/resources/some_weird_format.txt")
+    HdfsHelper.deleteFile(weirdFormatFilePath)
 
     val textContent = (
       "3 first line of the first record\n" +
@@ -87,16 +89,9 @@ class SparkHelperTest
         "4 another line for the third record"
     )
 
-    HdfsHelper
-      .writeToHdfsFile(textContent, "src/test/resources/some_weird_format.txt")
+    HdfsHelper.writeToHdfsFile(textContent, weirdFormatFilePath)
 
-    var computedRecords = SparkHelper
-      .textFileWithDelimiter(
-        "src/test/resources/some_weird_format.txt",
-        sc,
-        "\n3"
-      )
-      .collect()
+    var computedRecords = sc.textFile(weirdFormatFilePath, "\n3").collect()
 
     var expectedRecords = Array(
       (
@@ -113,13 +108,15 @@ class SparkHelperTest
 
     assert(computedRecords === expectedRecords)
 
-    HdfsHelper.deleteFile("src/test/resources/some_weird_format.txt")
+    HdfsHelper.deleteFile(weirdFormatFilePath)
 
     // 2: Let's read an xml file:
 
-    HdfsHelper.deleteFile("src/test/resources/some_basic_xml.xml")
+    val xmlFilePath = s"$resourceFolder/some_basic_xml.xml"
+
+    HdfsHelper.deleteFile(xmlFilePath)
 
-    val xmlTextContent = (
+    val xmlTextContent =
       "<Customers>\n" +
         "<Customer>\n" +
         "<Address>34 thingy street, someplace, sometown</Address>\n" +
@@ -128,18 +125,10 @@ class SparkHelperTest
         "<Address>12 thingy street, someplace, sometown</Address>\n" +
         "</Customer>\n" +
         "</Customers>"
-    )
 
-    HdfsHelper
-      .writeToHdfsFile(xmlTextContent, "src/test/resources/some_basic_xml.xml")
+    HdfsHelper.writeToHdfsFile(xmlTextContent, xmlFilePath)
 
-    computedRecords = SparkHelper
-      .textFileWithDelimiter(
-        "src/test/resources/some_basic_xml.xml",
-        sc,
-        "<Customer>\n"
-      )
-      .collect()
+    computedRecords = sc.textFile(xmlFilePath, "<Customer>\n").collect()
 
     expectedRecords = Array(
       "<Customers>\n",
@@ -156,7 +145,7 @@ class SparkHelperTest
 
     assert(computedRecords === expectedRecords)
 
-    HdfsHelper.deleteFile("src/test/resources/some_basic_xml.xml")
+    HdfsHelper.deleteFile(xmlFilePath)
   }
 
   test("Save as text file by key") {

From 04a089cdf8d5625281170d70866db4cad7ee9dd3 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 24 May 2018 22:47:44 +0100
Subject: [PATCH 05/25] Refactor saveAsTextFileByKey and place it in an
 implicit class

---
 .../scala/com/spark_helper/SparkHelper.scala  | 280 +++++++++++-------
 .../com/spark_helper/SparkHelperTest.scala    |  74 +++--
 2 files changed, 218 insertions(+), 136 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 12e0164..5cecf78 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -64,15 +64,14 @@ object SparkHelper extends Serializable {
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", classOf[BZip2Codec]) }}}
       *
       * @param outputFile the path of the produced file
-      * @param compressionCodec the type of compression to use (for instance
+      * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
       */
     def saveAsSingleTextFile(
         outputFile: String,
-        compressionCodec: Class[_ <: CompressionCodec]
+        codec: Class[_ <: CompressionCodec]
     ): Unit =
-      SparkHelper
-        .saveAsSingleTextFileInternal(rdd, outputFile, Some(compressionCodec))
+      SparkHelper.saveAsSingleTextFileInternal(rdd, outputFile, Some(codec))
 
     /** Saves an RDD in exactly one file.
       *
@@ -115,20 +114,116 @@ object SparkHelper extends Serializable {
       * @param outputFile the path of the produced file
       * @param workingFolder the path where file manipulations will temporarily
       * happen.
-      * @param compressionCodec the type of compression to use (for instance
+      * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
       */
     def saveAsSingleTextFile(
         outputFile: String,
         workingFolder: String,
-        compressionCodec: Class[_ <: CompressionCodec]
+        codec: Class[_ <: CompressionCodec]
     ): Unit =
       SparkHelper.saveAsSingleTextFileWithWorkingFolderInternal(
         rdd,
         outputFile,
         workingFolder,
-        Some(compressionCodec)
+        Some(codec)
       )
+
+  }
+
+  implicit class PairRDDExtensions(val rdd: RDD[(String, String)])
+      extends AnyVal {
+
+    /** Saves and repartitions a key/value RDD on files whose name is the key.
+      *
+      * Within the provided path, there will be one file per key in the given
+      * keyValueRDD. And within a file for a given key are only stored values
+      * for this key.
+      *
+      * As this internally needs to know the nbr of keys, this will have to
+      * compute it. If this nbr of keys is known beforehand, it would spare
+      * resources to use saveAsTextFileByKey(path: String, keyNbr: Int)
+      * instead.
+      *
+      * This is not scalable. This shouldn't be considered for any data flow
+      * with normal or big volumes.
+      *
+      * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path") }}}
+      *
+      * @param path the folder where will be storrred key files
+      */
+    def saveAsTextFileByKey(path: String): Unit =
+      SparkHelper.saveAsTextFileByKeyInternal(rdd, path, None, None)
+
+    /** Saves and repartitions a key/value RDD on files whose name is the key.
+      *
+      * Within the provided path, there will be one file per key in the given
+      * keyValueRDD. And within a file for a given key are only stored values
+      * for this key.
+      *
+      * This is not scalable. This shouldn't be considered for any data flow
+      * with normal or big volumes.
+      *
+      * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", 12) }}}
+      *
+      * @param path the folder where will be storrred key files
+      * @param keyNbr the nbr of expected keys (which is the nbr of outputed
+      * files)
+      */
+    def saveAsTextFileByKey(path: String, keyNbr: Int): Unit =
+      SparkHelper.saveAsTextFileByKeyInternal(rdd, path, Some(keyNbr), None)
+
+    /** Saves and repartitions a key/value RDD on files whose name is the key.
+      *
+      * Within the provided path, there will be one file per key in the given
+      * keyValueRDD. And within a file for a given key are only stored values
+      * for this key.
+      *
+      * As this internally needs to know the nbr of keys, this will have to
+      * compute it. If this nbr of keys is known beforehand, it would spare
+      * resources to use
+      * saveAsTextFileByKey(path: String, keyNbr: Int, codec: Class[_ <: CompressionCodec])
+      * instead.
+      *
+      * This is not scalable. This shouldn't be considered for any data flow
+      * with normal or big volumes.
+      *
+      * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", classOf[BZip2Codec]) }}}
+      *
+      * @param path the folder where will be storrred key files
+      * @param codec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def saveAsTextFileByKey(
+        path: String,
+        codec: Class[_ <: CompressionCodec]
+    ): Unit =
+      SparkHelper.saveAsTextFileByKeyInternal(rdd, path, None, Some(codec))
+
+    /** Saves and repartitions a key/value RDD on files whose name is the key.
+      *
+      * Within the provided path, there will be one file per key in the given
+      * keyValueRDD. And within a file for a given key are only stored values
+      * for this key.
+      *
+      * This is not scalable. This shouldn't be considered for any data flow
+      * with normal or big volumes.
+      *
+      * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", 12, classOf[BZip2Codec]) }}}
+      *
+      * @param path the folder where will be storrred key files
+      * @param keyNbr the nbr of expected keys (which is the nbr of outputed
+      * files)
+      * @param codec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def saveAsTextFileByKey(
+        path: String,
+        keyNbr: Int,
+        codec: Class[_ <: CompressionCodec]
+    ): Unit =
+      SparkHelper
+        .saveAsTextFileByKeyInternal(rdd, path, Some(keyNbr), Some(codec))
   }
 
   implicit class SparkContextExtensions(val sc: SparkContext) extends AnyVal {
@@ -210,92 +305,6 @@ object SparkHelper extends Serializable {
     }
   }
 
-  /** Saves and repartitions a key/value RDD on files whose name is the key.
-    *
-    * Within the provided outputFolder, will be one file per key in your
-    * keyValueRDD. And within a file for a given key are only values for this
-    * key.
-    *
-    * You need to know the nbr of keys beforehand (in general you use this to
-    * split your dataset in subsets, or to output one file per client, so you
-    * know how many keys you have). So you need to put as keyNbr the exact nbr
-    * of keys you'll have.
-    *
-    * This is not scalable. This shouldn't be considered for any data flow with
-    * normal or big volumes.
-    *
-    * {{{
-    * SparkHelper.saveAsTextFileByKey(
-    *   myKeyValueRddToStore, "/my/output/folder/path", 12)
-    * }}}
-    *
-    * @param keyValueRDD the key/value RDD
-    * @param outputFolder the foldder where will be storrred key files
-    * @param keyNbr the nbr of expected keys (which is the nbr of outputed files)
-    */
-  def saveAsTextFileByKey(
-      keyValueRDD: RDD[(String, String)],
-      outputFolder: String,
-      keyNbr: Int
-  ): Unit = {
-
-    HdfsHelper.deleteFolder(outputFolder)
-
-    keyValueRDD
-      .partitionBy(new HashPartitioner(keyNbr))
-      .saveAsHadoopFile(
-        outputFolder,
-        classOf[String],
-        classOf[String],
-        classOf[KeyBasedOutput]
-      )
-  }
-
-  /** Saves and repartitions a key/value RDD on files whose name is the key.
-    *
-    * Within the provided outputFolder, will be one file per key in your
-    * keyValueRDD. And within a file for a given key are only values for this
-    * key.
-    *
-    * You need to know the nbr of keys beforehand (in general you use this to
-    * split your dataset in subsets, or to output one file per client, so you
-    * know how many keys you have). So you need to put as keyNbr the exact nbr
-    * of keys you'll have.
-    *
-    * This is not scalable. This shouldn't be considered for any data flow with
-    * normal or big volumes.
-    *
-    * {{{
-    * SparkHelper.saveAsTextFileByKey(
-    *   myKeyValueRddToStore, "/my/output/folder/path", 12, classOf[BZip2Codec])
-    * }}}
-    *
-    * @param keyValueRDD the key/value RDD
-    * @param outputFolder the foldder where will be storrred key files
-    * @param keyNbr the nbr of expected keys (which is the nbr of outputed files)
-    * @param compressionCodec the type of compression to use (for instance
-    * classOf[BZip2Codec] or classOf[GzipCodec]))
-    */
-  def saveAsTextFileByKey(
-      keyValueRDD: RDD[(String, String)],
-      outputFolder: String,
-      keyNbr: Int,
-      compressionCodec: Class[_ <: CompressionCodec]
-  ): Unit = {
-
-    HdfsHelper.deleteFolder(outputFolder)
-
-    keyValueRDD
-      .partitionBy(new HashPartitioner(keyNbr))
-      .saveAsHadoopFile(
-        outputFolder,
-        classOf[String],
-        classOf[String],
-        classOf[KeyBasedOutput],
-        compressionCodec
-      )
-  }
-
   /** Decreases the nbr of partitions of a folder.
     *
     * This is often handy when the last step of your job needs to run on
@@ -363,7 +372,7 @@ object SparkHelper extends Serializable {
     * @param finalCoalescenceLevel the nbr of files within the folder at the end
     * of this method.
     * @param sparkContext the SparkContext
-    * @param compressionCodec the type of compression to use (for instance
+    * @param codec the type of compression to use (for instance
     * classOf[BZip2Codec] or classOf[GzipCodec]))
     */
   def decreaseCoalescence(
@@ -371,14 +380,15 @@ object SparkHelper extends Serializable {
       lowerCoalescenceLevelFolder: String,
       finalCoalescenceLevel: Int,
       sparkContext: SparkContext,
-      compressionCodec: Class[_ <: CompressionCodec]
+      codec: Class[_ <: CompressionCodec]
   ): Unit =
     decreaseCoalescenceInternal(
       highCoalescenceLevelFolder,
       lowerCoalescenceLevelFolder,
       finalCoalescenceLevel,
       sparkContext,
-      Some(compressionCodec))
+      Some(codec)
+    )
 
   /** Saves as text file, but by decreasing the nbr of partitions of the output.
     *
@@ -423,7 +433,8 @@ object SparkHelper extends Serializable {
       outputFolder,
       finalCoalescenceLevel,
       sparkContext,
-      None)
+      None
+    )
   }
 
   /** Saves as text file, but by decreasing the nbr of partitions of the output.
@@ -448,14 +459,14 @@ object SparkHelper extends Serializable {
     * finalCoalescenceLevel parameter).
     * @param finalCoalescenceLevel the nbr of files within the folder at the end
     * of this method.
-    * @param compressionCodec the type of compression to use (for instance
+    * @param codec the type of compression to use (for instance
     * classOf[BZip2Codec] or classOf[GzipCodec]))
     */
   def saveAsTextFileAndCoalesce(
       outputRDD: RDD[String],
       outputFolder: String,
       finalCoalescenceLevel: Int,
-      compressionCodec: Class[_ <: CompressionCodec]
+      codec: Class[_ <: CompressionCodec]
   ): Unit = {
 
     val sparkContext = outputRDD.context
@@ -475,7 +486,8 @@ object SparkHelper extends Serializable {
       outputFolder,
       finalCoalescenceLevel,
       sparkContext,
-      Some(compressionCodec))
+      Some(codec)
+    )
   }
 
   /** Equivalent to sparkContext.textFile(), but for each line is associated
@@ -539,7 +551,7 @@ object SparkHelper extends Serializable {
       outputRDD: RDD[String],
       outputFile: String,
       workingFolder: String,
-      compressionCodec: Option[Class[_ <: CompressionCodec]]
+      codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
     // We chose a random name for the temporary file:
@@ -547,7 +559,7 @@ object SparkHelper extends Serializable {
     val temporaryFile = s"$workingFolder/$temporaryName"
 
     // We perform the merge into a temporary single text file:
-    saveAsSingleTextFileInternal(outputRDD, temporaryFile, compressionCodec)
+    saveAsSingleTextFileInternal(outputRDD, temporaryFile, codec)
 
     // And then only we put the resulting file in its final real location:
     HdfsHelper.moveFile(temporaryFile, outputFile, overwrite = true)
@@ -565,7 +577,7 @@ object SparkHelper extends Serializable {
   private def saveAsSingleTextFileInternal(
       outputRDD: RDD[String],
       outputFile: String,
-      compressionCodec: Option[Class[_ <: CompressionCodec]]
+      codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
     val hadoopConfiguration = outputRDD.sparkContext.hadoopConfiguration
@@ -573,9 +585,9 @@ object SparkHelper extends Serializable {
 
     // Classic saveAsTextFile in a temporary folder:
     HdfsHelper.deleteFolder(s"$outputFile.tmp")
-    compressionCodec match {
-      case Some(compressionCodec) =>
-        outputRDD.saveAsTextFile(s"$outputFile.tmp", compressionCodec)
+    codec match {
+      case Some(codec) =>
+        outputRDD.saveAsTextFile(s"$outputFile.tmp", codec)
       case None =>
         outputRDD.saveAsTextFile(s"$outputFile.tmp")
     }
@@ -593,22 +605,68 @@ object SparkHelper extends Serializable {
     HdfsHelper.deleteFolder(s"$outputFile.tmp")
   }
 
+  private def saveAsTextFileByKeyInternal(
+      rdd: RDD[(String, String)],
+      path: String,
+      optKeyNbr: Option[Int],
+      codec: Option[Class[_ <: CompressionCodec]]
+  ): Unit = {
+
+    HdfsHelper.deleteFolder(path)
+
+    // Whether the rdd was already cached or not (used to unpersist it if we
+    // have to get the nbr of keys):
+    val isCached = rdd.getStorageLevel.useMemory
+
+    // If the nbr of keys isn't provided, we have to get it ourselves:
+    val keyNbr = optKeyNbr match {
+      case Some(keyNbr) =>
+        keyNbr
+      case None =>
+        if (!isCached)
+          rdd.cache()
+        rdd.keys.distinct.count.toInt
+    }
+
+    val prdd = rdd.partitionBy(new HashPartitioner(keyNbr))
+
+    codec match {
+      case Some(codec) =>
+        prdd.saveAsHadoopFile(
+          path,
+          classOf[String],
+          classOf[String],
+          classOf[KeyBasedOutput],
+          codec
+        )
+      case None =>
+        prdd.saveAsHadoopFile(
+          path,
+          classOf[String],
+          classOf[String],
+          classOf[KeyBasedOutput]
+        )
+    }
+
+    if (optKeyNbr.isEmpty && !isCached)
+      rdd.unpersist()
+  }
+
   private def decreaseCoalescenceInternal(
       highCoalescenceLevelFolder: String,
       lowerCoalescenceLevelFolder: String,
       finalCoalescenceLevel: Int,
       sparkContext: SparkContext,
-      compressionCodec: Option[Class[_ <: CompressionCodec]]
+      codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
     val intermediateRDD = sparkContext
       .textFile(highCoalescenceLevelFolder)
       .coalesce(finalCoalescenceLevel)
 
-    compressionCodec match {
-      case Some(compressionCodec) =>
-        intermediateRDD
-          .saveAsTextFile(lowerCoalescenceLevelFolder, compressionCodec)
+    codec match {
+      case Some(codec) =>
+        intermediateRDD.saveAsTextFile(lowerCoalescenceLevelFolder, codec)
       case None =>
         intermediateRDD.saveAsTextFile(lowerCoalescenceLevelFolder)
     }
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index c476928..2fbcdf0 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,6 +1,7 @@
 package com.spark_helper
 
-import com.spark_helper.SparkHelper.{RDDExtensions, SparkContextExtensions}
+import com.spark_helper.SparkHelper.{RDDExtensions, PairRDDExtensions}
+import com.spark_helper.SparkHelper.SparkContextExtensions
 
 import org.apache.hadoop.io.compress.GzipCodec
 
@@ -150,7 +151,11 @@ class SparkHelperTest
 
   test("Save as text file by key") {
 
-    HdfsHelper.deleteFolder("src/test/resources/key_value_storage")
+    val keyValueFolder = s"$resourceFolder/key_value_storage"
+
+    // 1: Let's strore key values per file:
+
+    HdfsHelper.deleteFolder(keyValueFolder)
 
     val someKeyValueRdd = sc.parallelize[(String, String)](
       Array(
@@ -164,42 +169,61 @@ class SparkHelperTest
       )
     )
 
-    SparkHelper.saveAsTextFileByKey(
-      someKeyValueRdd,
-      "src/test/resources/key_value_storage",
-      3)
+    someKeyValueRdd.saveAsTextFileByKey(keyValueFolder, 3)
 
     // The folder key_value_storage has been created:
-    assert(HdfsHelper.folderExists("src/test/resources/key_value_storage"))
+    assert(HdfsHelper.folderExists(keyValueFolder))
 
     // And it contains one file per key:
-    val genratedKeyFiles = HdfsHelper
-      .listFileNamesInFolder("src/test/resources/key_value_storage")
-    val expectedKeyFiles = List("_SUCCESS", "key_1", "key_2", "key_3")
+    var genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    var expectedKeyFiles = List("_SUCCESS", "key_1", "key_2", "key_3")
     assert(genratedKeyFiles === expectedKeyFiles)
 
-    val valuesForKey1 = sc
-      .textFile("src/test/resources/key_value_storage/key_1")
-      .collect()
-      .sorted
+    var valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1").collect().sorted
+    assert(valuesForKey1 === Array("value_a", "value_b"))
+
+    val valuesForKey2 = sc.textFile(s"$keyValueFolder/key_2").collect().sorted
+    assert(valuesForKey2 === Array("value_b", "value_c", "value_d"))
+
+    val valuesForKey3 = sc.textFile(s"$keyValueFolder/key_3").collect().sorted
+    assert(valuesForKey3 === Array("value_a", "value_b"))
+
+    // 2: Let's strore key values per file; but without providing the nbr of
+    // keys:
+
+    HdfsHelper.deleteFolder(keyValueFolder)
+
+    someKeyValueRdd.saveAsTextFileByKey(keyValueFolder)
+
+    // The folder key_value_storage has been created:
+    assert(HdfsHelper.folderExists(keyValueFolder))
 
+    // And it contains one file per key:
+    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    expectedKeyFiles = List("_SUCCESS", "key_1", "key_2", "key_3")
+    assert(genratedKeyFiles === expectedKeyFiles)
+
+    valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1").collect().sorted
     assert(valuesForKey1 === Array("value_a", "value_b"))
 
-    val valuesForKey2 = sc
-      .textFile("src/test/resources/key_value_storage/key_2")
-      .collect()
-      .sorted
+    // 3: Let's strore key values per file and compress these files:
 
-    assert(valuesForKey2 === Array("value_b", "value_c", "value_d"))
+    HdfsHelper.deleteFolder(keyValueFolder)
 
-    val valuesForKey3 = sc
-      .textFile("src/test/resources/key_value_storage/key_3")
-      .collect()
-      .sorted
+    someKeyValueRdd.saveAsTextFileByKey(keyValueFolder, 3, classOf[GzipCodec])
 
-    assert(valuesForKey3 === Array("value_a", "value_b"))
+    // The folder key_value_storage has been created:
+    assert(HdfsHelper.folderExists(keyValueFolder))
+
+    // And it contains one file per key:
+    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    expectedKeyFiles = List("_SUCCESS", "key_1.gz", "key_2.gz", "key_3.gz")
+    assert(genratedKeyFiles === expectedKeyFiles)
+
+    valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1.gz").collect().sorted
+    assert(valuesForKey1 === Array("value_a", "value_b"))
 
-    HdfsHelper.deleteFolder("src/test/resources/key_value_storage")
+    HdfsHelper.deleteFolder(keyValueFolder)
   }
 
   test("Decrease coalescence level") {

From d756f0288a99f67efe081e529c9b49178f47fa62 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sat, 26 May 2018 11:02:26 +0100
Subject: [PATCH 06/25] Test saveAsTextFileAndCoalesce and place it in an
 implicit class

---
 .../scala/com/spark_helper/SparkHelper.scala  | 270 +++++++++---------
 .../com/spark_helper/SparkHelperTest.scala    |  40 +++
 2 files changed, 171 insertions(+), 139 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 5cecf78..8c253d4 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -51,10 +51,10 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt") }}}
       *
-      * @param outputFile the path of the produced file
+      * @param path the path of the produced file
       */
-    def saveAsSingleTextFile(outputFile: String): Unit =
-      SparkHelper.saveAsSingleTextFileInternal(rdd, outputFile, None)
+    def saveAsSingleTextFile(path: String): Unit =
+      SparkHelper.saveAsSingleTextFileInternal(rdd, path, None)
 
     /** Saves an RDD in exactly one file.
       *
@@ -63,15 +63,15 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", classOf[BZip2Codec]) }}}
       *
-      * @param outputFile the path of the produced file
+      * @param path the path of the produced file
       * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
       */
     def saveAsSingleTextFile(
-        outputFile: String,
+        path: String,
         codec: Class[_ <: CompressionCodec]
     ): Unit =
-      SparkHelper.saveAsSingleTextFileInternal(rdd, outputFile, Some(codec))
+      SparkHelper.saveAsSingleTextFileInternal(rdd, path, Some(codec))
 
     /** Saves an RDD in exactly one file.
       *
@@ -85,14 +85,14 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path") }}}
       *
-      * @param outputFile the path of the produced file
+      * @param path the path of the produced file
       * @param workingFolder the path where file manipulations will temporarily
       * happen.
       */
-    def saveAsSingleTextFile(outputFile: String, workingFolder: String): Unit =
+    def saveAsSingleTextFile(path: String, workingFolder: String): Unit =
       SparkHelper.saveAsSingleTextFileWithWorkingFolderInternal(
         rdd,
-        outputFile,
+        path,
         workingFolder,
         None
       )
@@ -111,24 +111,116 @@ object SparkHelper extends Serializable {
       * rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path", classOf[BZip2Codec])
       * }}}
       *
-      * @param outputFile the path of the produced file
+      * @param path the path of the produced file
       * @param workingFolder the path where file manipulations will temporarily
       * happen.
       * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
       */
     def saveAsSingleTextFile(
-        outputFile: String,
+        path: String,
         workingFolder: String,
         codec: Class[_ <: CompressionCodec]
     ): Unit =
       SparkHelper.saveAsSingleTextFileWithWorkingFolderInternal(
         rdd,
-        outputFile,
+        path,
         workingFolder,
         Some(codec)
       )
 
+    /** Saves as text file, but by decreasing the nbr of partitions of the output.
+      *
+      * Same as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.saveAsTextFile()</code>
+      * , but decreases the nbr of partitions in the output folder before doing
+      * so.
+      *
+      * The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
+      * , but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
+      * is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
+      * would make the processing time explode, wherease this methods keeps the
+      * processing parallelized, save as text file and then only merges the
+      * result in a lower nbr of partitions.
+      *
+      * {{{ rdd.saveAsTextFileAndCoalesce("/produced/folder/path/with/only/30/files", 30) }}}
+      *
+      * @param path the folder where will finally be stored the RDD but spread
+      * on only 30 files (where 30 is the value of the finalCoalesceLevel
+      * parameter).
+      * @param finalCoalesceLevel the nbr of files within the folder at the end
+      * of this method.
+      */
+    def saveAsTextFileAndCoalesce(
+        path: String,
+        finalCoalesceLevel: Int
+    ): Unit = {
+
+      // We remove folders where to store data in case they already exist:
+      HdfsHelper.deleteFolder(s"${path}_tmp")
+      HdfsHelper.deleteFolder(path)
+
+      // We first save the rdd with the level of coalescence used during the
+      // processing. This way the processing is done with the right level of
+      // tasks:
+      rdd.saveAsTextFile(s"${path}_tmp")
+
+      // Then we read back this tmp folder, apply the coalesce and store it back:
+      SparkHelper.decreaseCoalescenceInternal(
+        s"${path}_tmp",
+        path,
+        finalCoalesceLevel,
+        rdd.context,
+        None
+      )
+    }
+
+    /** Saves as text file, but by decreasing the nbr of partitions of the output.
+      *
+      * Same as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.saveAsTextFile()</code>
+      * , but decreases the nbr of partitions in the output folder before doing
+      * so.
+      *
+      * The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
+      * , but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
+      * is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
+      * would make the processing time explode, wherease this methods keeps the
+      * processing parallelized, save as text file and then only merges the
+      * result in a lower nbr of partitions.
+      *
+      * {{{ rdd.saveAsTextFileAndCoalesce("/produced/folder/path/with/only/30/files", 30, classOf[BZip2Codec]) }}}
+      *
+      * @param path the folder where will finally be stored the RDD but spread
+      * on only 30 files (where 30 is the value of the finalCoalesceLevel
+      * parameter).
+      * @param finalCoalesceLevel the nbr of files within the folder at the end
+      * of this method.
+      * @param codec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def saveAsTextFileAndCoalesce(
+        path: String,
+        finalCoalesceLevel: Int,
+        codec: Class[_ <: CompressionCodec]
+    ): Unit = {
+
+      // We remove folders where to store data in case they already exist:
+      HdfsHelper.deleteFolder(s"${path}_tmp")
+      HdfsHelper.deleteFolder(path)
+
+      // We first save the rdd with the level of coalescence used during the
+      // processing. This way the processing is done with the right level of
+      // tasks:
+      rdd.saveAsTextFile(s"${path}_tmp")
+
+      // Then we read back this tmp folder, apply the coalesce and store it back:
+      decreaseCoalescenceInternal(
+        s"${path}_tmp",
+        path,
+        finalCoalesceLevel,
+        rdd.context,
+        Some(codec)
+      )
+    }
   }
 
   implicit class PairRDDExtensions(val rdd: RDD[(String, String)])
@@ -268,7 +360,7 @@ object SparkHelper extends Serializable {
       * assert(computedRecords == expectedRecords)
       * }}}
       *
-      * @param hdfsPath the path of the file to read (folder or file, '*' works
+      * @param path the path of the file to read (folder or file, '*' works
       * as well).
       * @param delimiter the specific record delimiter which replaces "\n"
       * @param maxRecordLength the max length (not sure which unit) of a record
@@ -276,7 +368,7 @@ object SparkHelper extends Serializable {
       * @return the RDD of records
       */
     def textFile(
-        hdfsPath: String,
+        path: String,
         delimiter: String,
         maxRecordLength: String = "1000000"
     ): RDD[String] = {
@@ -295,7 +387,7 @@ object SparkHelper extends Serializable {
         .set("mapreduce.input.linerecordreader.line.maxlength", maxRecordLength)
 
       sc.newAPIHadoopFile(
-          hdfsPath,
+          path,
           classOf[TextInputFormat],
           classOf[LongWritable],
           classOf[Text],
@@ -327,21 +419,21 @@ object SparkHelper extends Serializable {
     * @param highCoalescenceLevelFolder the folder which contains 10000 files
     * @param lowerCoalescenceLevelFolder the folder which will contain the same
     * data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-    * is the finalCoalescenceLevel parameter).
-    * @param finalCoalescenceLevel the nbr of files within the folder at the end
+    * is the finalCoalesceLevel parameter).
+    * @param finalCoalesceLevel the nbr of files within the folder at the end
     * of this method.
     * @param sparkContext the SparkContext
     */
   def decreaseCoalescence(
       highCoalescenceLevelFolder: String,
       lowerCoalescenceLevelFolder: String,
-      finalCoalescenceLevel: Int,
+      finalCoalesceLevel: Int,
       sparkContext: SparkContext
   ): Unit =
     decreaseCoalescenceInternal(
       highCoalescenceLevelFolder,
       lowerCoalescenceLevelFolder,
-      finalCoalescenceLevel,
+      finalCoalesceLevel,
       sparkContext,
       None)
 
@@ -368,8 +460,8 @@ object SparkHelper extends Serializable {
     * @param highCoalescenceLevelFolder the folder which contains 10000 files
     * @param lowerCoalescenceLevelFolder the folder which will contain the same
     * data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-    * is the finalCoalescenceLevel parameter).
-    * @param finalCoalescenceLevel the nbr of files within the folder at the end
+    * is the finalCoalesceLevel parameter).
+    * @param finalCoalesceLevel the nbr of files within the folder at the end
     * of this method.
     * @param sparkContext the SparkContext
     * @param codec the type of compression to use (for instance
@@ -378,117 +470,17 @@ object SparkHelper extends Serializable {
   def decreaseCoalescence(
       highCoalescenceLevelFolder: String,
       lowerCoalescenceLevelFolder: String,
-      finalCoalescenceLevel: Int,
+      finalCoalesceLevel: Int,
       sparkContext: SparkContext,
       codec: Class[_ <: CompressionCodec]
   ): Unit =
     decreaseCoalescenceInternal(
       highCoalescenceLevelFolder,
       lowerCoalescenceLevelFolder,
-      finalCoalescenceLevel,
-      sparkContext,
-      Some(codec)
-    )
-
-  /** Saves as text file, but by decreasing the nbr of partitions of the output.
-    *
-    * Same as decreaseCoalescence, but the storage of the RDD in an intermediate
-    * folder is included.
-    *
-    * This still makes the processing parallelized, but the output is coalesced.
-    *
-    * {{{
-    * SparkHelper.saveAsTextFileAndCoalesce(
-    *   myRddToStore, "/produced/folder/path/with/only/300/files", 300)
-    * }}}
-    *
-    * @param outputRDD the RDD to store, processed for instance on 10000 tasks
-    * (which would thus be stored as 10000 files).
-    * @param outputFolder the folder where will finally be stored the RDD but
-    * spread on only 300 files (where 300 is the value of the
-    * finalCoalescenceLevel parameter).
-    * @param finalCoalescenceLevel the nbr of files within the folder at the end
-    * of this method.
-    */
-  def saveAsTextFileAndCoalesce(
-      outputRDD: RDD[String],
-      outputFolder: String,
-      finalCoalescenceLevel: Int
-  ): Unit = {
-
-    val sparkContext = outputRDD.context
-
-    // We remove folders where to store data in case they already exist:
-    HdfsHelper.deleteFolder(outputFolder + "_tmp")
-    HdfsHelper.deleteFolder(outputFolder)
-
-    // We first save the rdd with the level of coalescence used during the
-    // processing. This way the processing is done with the right level of
-    // tasks:
-    outputRDD.saveAsTextFile(outputFolder + "_tmp")
-
-    // Then we read back this tmp folder, apply the coalesce and store it back:
-    decreaseCoalescenceInternal(
-      outputFolder + "_tmp",
-      outputFolder,
-      finalCoalescenceLevel,
-      sparkContext,
-      None
-    )
-  }
-
-  /** Saves as text file, but by decreasing the nbr of partitions of the output.
-    *
-    * Same as decreaseCoalescence, but the storage of the RDD in an intermediate
-    * folder is included.
-    *
-    * This still makes the processing parallelized, but the output is coalesced.
-    *
-    * {{{
-    * SparkHelper.saveAsTextFileAndCoalesce(
-    *   myRddToStore,
-    *   "/produced/folder/path/with/only/300/files",
-    *   300,
-    *   classOf[BZip2Codec])
-    * }}}
-    *
-    * @param outputRDD the RDD to store, processed for instance on 10000 tasks
-    * (which would thus be stored as 10000 files).
-    * @param outputFolder the folder where will finally be stored the RDD but
-    * spread on only 300 files (where 300 is the value of the
-    * finalCoalescenceLevel parameter).
-    * @param finalCoalescenceLevel the nbr of files within the folder at the end
-    * of this method.
-    * @param codec the type of compression to use (for instance
-    * classOf[BZip2Codec] or classOf[GzipCodec]))
-    */
-  def saveAsTextFileAndCoalesce(
-      outputRDD: RDD[String],
-      outputFolder: String,
-      finalCoalescenceLevel: Int,
-      codec: Class[_ <: CompressionCodec]
-  ): Unit = {
-
-    val sparkContext = outputRDD.context
-
-    // We remove folders where to store data in case they already exist:
-    HdfsHelper.deleteFolder(outputFolder + "_tmp")
-    HdfsHelper.deleteFolder(outputFolder)
-
-    // We first save the rdd with the level of coalescence used during the
-    // processing. This way the processing is done with the right level of
-    // tasks:
-    outputRDD.saveAsTextFile(outputFolder + "_tmp")
-
-    // Then we read back this tmp folder, apply the coalesce and store it back:
-    decreaseCoalescenceInternal(
-      outputFolder + "_tmp",
-      outputFolder,
-      finalCoalescenceLevel,
+      finalCoalesceLevel,
       sparkContext,
       Some(codec)
     )
-  }
 
   /** Equivalent to sparkContext.textFile(), but for each line is associated
     * with its file path.
@@ -515,13 +507,13 @@ object SparkHelper extends Serializable {
     * )
     * }}}
     *
-    * @param hdfsPath the path of the folder (or structure of folders) to read
+    * @param path the path of the folder (or structure of folders) to read
     * @param sparkContext the SparkContext
     * @return the RDD of records where a record is a tuple containing the path
     * of the file the record comes from and the record itself.
     */
   def textFileWithFileName(
-      hdfsPath: String,
+      path: String,
       sparkContext: SparkContext
   ): RDD[(String, String)] = {
 
@@ -531,7 +523,7 @@ object SparkHelper extends Serializable {
 
     sparkContext
       .hadoopFile(
-        hdfsPath,
+        path,
         classOf[TextInputFormat2],
         classOf[LongWritable],
         classOf[Text],
@@ -549,7 +541,7 @@ object SparkHelper extends Serializable {
 
   private def saveAsSingleTextFileWithWorkingFolderInternal(
       outputRDD: RDD[String],
-      outputFile: String,
+      path: String,
       workingFolder: String,
       codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
@@ -562,7 +554,7 @@ object SparkHelper extends Serializable {
     saveAsSingleTextFileInternal(outputRDD, temporaryFile, codec)
 
     // And then only we put the resulting file in its final real location:
-    HdfsHelper.moveFile(temporaryFile, outputFile, overwrite = true)
+    HdfsHelper.moveFile(temporaryFile, path, overwrite = true)
   }
 
   /** Saves RDD in exactly one file.
@@ -571,12 +563,12 @@ object SparkHelper extends Serializable {
     * the processing parallelized.
     *
     * @param outputRDD the RDD of strings to save as text file
-    * @param outputFile the path where to save the file
+    * @param path the path where to save the file
     * @param compression the compression codec to use (can be left to None)
     */
   private def saveAsSingleTextFileInternal(
       outputRDD: RDD[String],
-      outputFile: String,
+      path: String,
       codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
@@ -584,25 +576,25 @@ object SparkHelper extends Serializable {
     val fileSystem = FileSystem.get(hadoopConfiguration)
 
     // Classic saveAsTextFile in a temporary folder:
-    HdfsHelper.deleteFolder(s"$outputFile.tmp")
+    HdfsHelper.deleteFolder(s"$path.tmp")
     codec match {
       case Some(codec) =>
-        outputRDD.saveAsTextFile(s"$outputFile.tmp", codec)
+        outputRDD.saveAsTextFile(s"$path.tmp", codec)
       case None =>
-        outputRDD.saveAsTextFile(s"$outputFile.tmp")
+        outputRDD.saveAsTextFile(s"$path.tmp")
     }
 
     // Merge the folder into a single file:
-    HdfsHelper.deleteFile(outputFile)
+    HdfsHelper.deleteFile(path)
     FileUtil.copyMerge(
       fileSystem,
-      new Path(s"$outputFile.tmp"),
+      new Path(s"$path.tmp"),
       fileSystem,
-      new Path(outputFile),
+      new Path(path),
       true,
       hadoopConfiguration,
       null)
-    HdfsHelper.deleteFolder(s"$outputFile.tmp")
+    HdfsHelper.deleteFolder(s"$path.tmp")
   }
 
   private def saveAsTextFileByKeyInternal(
@@ -655,14 +647,14 @@ object SparkHelper extends Serializable {
   private def decreaseCoalescenceInternal(
       highCoalescenceLevelFolder: String,
       lowerCoalescenceLevelFolder: String,
-      finalCoalescenceLevel: Int,
+      finalCoalesceLevel: Int,
       sparkContext: SparkContext,
       codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
     val intermediateRDD = sparkContext
       .textFile(highCoalescenceLevelFolder)
-      .coalesce(finalCoalescenceLevel)
+      .coalesce(finalCoalesceLevel)
 
     codec match {
       case Some(codec) =>
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 2fbcdf0..478aad5 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -226,6 +226,46 @@ class SparkHelperTest
     HdfsHelper.deleteFolder(keyValueFolder)
   }
 
+  test("Save as text file and reduce nbr of partitions") {
+
+    val testFolder = s"$resourceFolder/folder"
+
+    HdfsHelper.deleteFolder(testFolder)
+
+    val rddToStore =
+      sc.parallelize(Array("data_a", "data_b", "data_c")).repartition(3)
+
+    // 1: Without compressing:
+
+    rddToStore.saveAsTextFileAndCoalesce(testFolder, 2)
+
+    // Let's check the nbr of partitions:
+    var genratedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
+    var expectedKeyFiles = List("_SUCCESS", "part-00000", "part-00001")
+    assert(genratedKeyFiles === expectedKeyFiles)
+
+    // And let's check the content:
+    var singleFileStoredData = sc.textFile(testFolder).collect().sorted
+    assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
+
+    HdfsHelper.deleteFolder(testFolder)
+
+    // 2: By compressing:
+
+    rddToStore.saveAsTextFileAndCoalesce(testFolder, 2, classOf[GzipCodec])
+
+    // Let's check the nbr of partitions:
+    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
+    expectedKeyFiles = List("_SUCCESS", "part-00000.gz", "part-00001.gz")
+    assert(genratedKeyFiles === expectedKeyFiles)
+
+    // And let's check the content:
+    singleFileStoredData = sc.textFile(testFolder).collect().sorted
+    assert(singleFileStoredData === Array("data_a", "data_b", "data_c"))
+
+    HdfsHelper.deleteFolder(testFolder)
+  }
+
   test("Decrease coalescence level") {
 
     HdfsHelper.deleteFolder("src/test/resources/re_coalescence_test_input")

From fdc854429b909f3e2180cec5947af996f4b8f135 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sat, 26 May 2018 12:12:15 +0100
Subject: [PATCH 07/25] Place decreaseCoalescence in the SparkContextExtensions
 implicit class

---
 .../scala/com/spark_helper/SparkHelper.scala  | 167 +++++++++---------
 .../com/spark_helper/SparkHelperTest.scala    |  43 ++---
 2 files changed, 98 insertions(+), 112 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 8c253d4..c5658f5 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -395,92 +395,89 @@ object SparkHelper extends Serializable {
         )
         .map { case (_, text) => text.toString }
     }
-  }
 
-  /** Decreases the nbr of partitions of a folder.
-    *
-    * This is often handy when the last step of your job needs to run on
-    * thousands of files, but you want to store your final output on let's say
-    * only 300 files.
-    *
-    * It's like a FileUtil.copyMerge, but the merging produces more than one
-    * file.
-    *
-    * Be aware that this methods deletes the provided input folder.
-    *
-    * {{{
-    * SparkHelper.decreaseCoalescence(
-    *   "/folder/path/with/2000/files",
-    *   "/produced/folder/path/with/only/300/files",
-    *   300,
-    *   sparkContext)
-    * }}}
-    *
-    * @param highCoalescenceLevelFolder the folder which contains 10000 files
-    * @param lowerCoalescenceLevelFolder the folder which will contain the same
-    * data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-    * is the finalCoalesceLevel parameter).
-    * @param finalCoalesceLevel the nbr of files within the folder at the end
-    * of this method.
-    * @param sparkContext the SparkContext
-    */
-  def decreaseCoalescence(
-      highCoalescenceLevelFolder: String,
-      lowerCoalescenceLevelFolder: String,
-      finalCoalesceLevel: Int,
-      sparkContext: SparkContext
-  ): Unit =
-    decreaseCoalescenceInternal(
-      highCoalescenceLevelFolder,
-      lowerCoalescenceLevelFolder,
-      finalCoalesceLevel,
-      sparkContext,
-      None)
-
-  /** Decreases the nbr of partitions of a folder.
-    *
-    * This is often handy when the last step of your job needs to run on
-    * thousands of files, but you want to store your final output on let's say
-    * only 300 files.
-    *
-    * It's like a FileUtil.copyMerge, but the merging produces more than one
-    * file.
-    *
-    * Be aware that this methods deletes the provided input folder.
-    *
-    * {{{
-    * SparkHelper.decreaseCoalescence(
-    *   "/folder/path/with/2000/files",
-    *   "/produced/folder/path/with/only/300/files",
-    *   300,
-    *   sparkContext,
-    *   classOf[BZip2Codec])
-    * }}}
-    *
-    * @param highCoalescenceLevelFolder the folder which contains 10000 files
-    * @param lowerCoalescenceLevelFolder the folder which will contain the same
-    * data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-    * is the finalCoalesceLevel parameter).
-    * @param finalCoalesceLevel the nbr of files within the folder at the end
-    * of this method.
-    * @param sparkContext the SparkContext
-    * @param codec the type of compression to use (for instance
-    * classOf[BZip2Codec] or classOf[GzipCodec]))
-    */
-  def decreaseCoalescence(
-      highCoalescenceLevelFolder: String,
-      lowerCoalescenceLevelFolder: String,
-      finalCoalesceLevel: Int,
-      sparkContext: SparkContext,
-      codec: Class[_ <: CompressionCodec]
-  ): Unit =
-    decreaseCoalescenceInternal(
-      highCoalescenceLevelFolder,
-      lowerCoalescenceLevelFolder,
-      finalCoalesceLevel,
-      sparkContext,
-      Some(codec)
-    )
+    /** Decreases the nbr of partitions of a folder.
+      *
+      * This comes in handy when the last step of your job needs to run on
+      * thousands of files, but you want to store your final output on let's say
+      * only 30 files.
+      *
+      * It's like a <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileUtil.copyMerge()</code>
+      * , but the merging produces more than one file.
+      *
+      * Be aware that this methods deletes the provided input folder.
+      *
+      * {{{
+      * sc.decreaseCoalescence(
+      *   "/folder/path/with/2000/files",
+      *   "/produced/folder/path/with/only/30/files",
+      *   30
+      * )
+      * }}}
+      *
+      * @param highCoalescenceLevelFolder the folder which contains 10000 files
+      * @param lowerCoalescenceLevelFolder the folder which will contain the same
+      * data as highCoalescenceLevelFolder but spread on only 30 files (where 30
+      * is the finalCoalesceLevel parameter).
+      * @param finalCoalesceLevel the nbr of files within the folder at the end
+      * of this method.
+      */
+    def decreaseCoalescence(
+        highCoalescenceLevelFolder: String,
+        lowerCoalescenceLevelFolder: String,
+        finalCoalesceLevel: Int
+    ): Unit =
+      SparkHelper.decreaseCoalescenceInternal(
+        highCoalescenceLevelFolder,
+        lowerCoalescenceLevelFolder,
+        finalCoalesceLevel,
+        sc,
+        None
+      )
+
+    /** Decreases the nbr of partitions of a folder.
+      *
+      * This comes in handy when the last step of your job needs to run on
+      * thousands of files, but you want to store your final output on let's say
+      * only 30 files.
+      *
+      * It's like a <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileUtil.copyMerge()</code>
+      * , but the merging produces more than one file.
+      *
+      * Be aware that this methods deletes the provided input folder.
+      *
+      * {{{
+      * sc.decreaseCoalescence(
+      *   "/folder/path/with/2000/files",
+      *   "/produced/folder/path/with/only/30/files",
+      *   30,
+      *   classOf[BZip2Codec]
+      * )
+      * }}}
+      *
+      * @param highCoalescenceLevelFolder the folder which contains 10000 files
+      * @param lowerCoalescenceLevelFolder the folder which will contain the same
+      * data as highCoalescenceLevelFolder but spread on only 30 files (where 30
+      * is the finalCoalesceLevel parameter).
+      * @param finalCoalesceLevel the nbr of files within the folder at the end
+      * of this method.
+      * @param codec the type of compression to use (for instance
+      * classOf[BZip2Codec] or classOf[GzipCodec]))
+      */
+    def decreaseCoalescence(
+        highCoalescenceLevelFolder: String,
+        lowerCoalescenceLevelFolder: String,
+        finalCoalesceLevel: Int,
+        codec: Class[_ <: CompressionCodec]
+    ): Unit =
+      SparkHelper.decreaseCoalescenceInternal(
+        highCoalescenceLevelFolder,
+        lowerCoalescenceLevelFolder,
+        finalCoalesceLevel,
+        sc,
+        Some(codec)
+      )
+  }
 
   /** Equivalent to sparkContext.textFile(), but for each line is associated
     * with its file path.
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 478aad5..2af3b4d 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -268,42 +268,30 @@ class SparkHelperTest
 
   test("Decrease coalescence level") {
 
-    HdfsHelper.deleteFolder("src/test/resources/re_coalescence_test_input")
-    HdfsHelper.deleteFolder("src/test/resources/re_coalescence_test_output")
+    val inputTestFolder = s"$resourceFolder/re_coalescence_test_input"
+    val outputTestFolder = s"$resourceFolder/re_coalescence_test_output"
+
+    HdfsHelper.deleteFolder(inputTestFolder)
+    HdfsHelper.deleteFolder(outputTestFolder)
 
     // Let's create the folder with high level of coalescence (3 files):
-    sc.parallelize[String](Array("data_1_a", "data_1_b", "data_1_c"))
-      .saveAsSingleTextFile(
-        "src/test/resources/re_coalescence_test_input/input_file_1"
-      )
-    sc.parallelize[String](Array("data_2_a", "data_2_b"))
-      .saveAsSingleTextFile(
-        "src/test/resources/re_coalescence_test_input/input_file_2"
-      )
-    sc.parallelize[String](Array("data_3_a", "data_3_b", "data_3_c"))
-      .saveAsSingleTextFile(
-        "src/test/resources/re_coalescence_test_input/input_file_3"
-      )
+    sc.parallelize(Array("data_1_a", "data_1_b", "data_1_c"))
+      .saveAsSingleTextFile(s"$inputTestFolder/input_file_1")
+    sc.parallelize(Array("data_2_a", "data_2_b"))
+      .saveAsSingleTextFile(s"$inputTestFolder/input_file_2")
+    sc.parallelize(Array("data_3_a", "data_3_b", "data_3_c"))
+      .saveAsSingleTextFile(s"$inputTestFolder/input_file_3")
 
     // Let's decrease the coalescence level in order to only have 2 files:
-    SparkHelper.decreaseCoalescence(
-      "src/test/resources/re_coalescence_test_input",
-      "src/test/resources/re_coalescence_test_output",
-      2,
-      sc)
+    sc.decreaseCoalescence(inputTestFolder, outputTestFolder, 2)
 
     // And we check we have two files in output:
-    val outputFileList = HdfsHelper
-      .listFileNamesInFolder("src/test/resources/re_coalescence_test_output")
+    val outputFileList = HdfsHelper.listFileNamesInFolder(outputTestFolder)
     val expectedFileList = List("_SUCCESS", "part-00000", "part-00001")
     assert(outputFileList === expectedFileList)
 
     // And that all input data is in the output:
-    val outputData = sc
-      .textFile("src/test/resources/re_coalescence_test_output")
-      .collect
-      .sorted
-
+    val outputData = sc.textFile(outputTestFolder).collect.sorted
     val expectedOutputData = Array(
       "data_1_a",
       "data_1_b",
@@ -316,7 +304,8 @@ class SparkHelperTest
     )
     assert(outputData === expectedOutputData)
 
-    HdfsHelper.deleteFolder("src/test/resources/re_coalescence_test_output")
+    HdfsHelper.deleteFolder(inputTestFolder)
+    HdfsHelper.deleteFolder(outputTestFolder)
   }
 
   test(

From 46318ef2a6b06b6e189549e369d958f586a05fee Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sat, 26 May 2018 12:30:07 +0100
Subject: [PATCH 08/25] Place textFileWithFileName in the
 SparkContextExtensions implicit class

---
 .../scala/com/spark_helper/SparkHelper.scala  | 105 +++++++++---------
 .../com/spark_helper/SparkHelperTest.scala    |  20 ++--
 2 files changed, 63 insertions(+), 62 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index c5658f5..2ed582e 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -396,6 +396,56 @@ object SparkHelper extends Serializable {
         .map { case (_, text) => text.toString }
     }
 
+    /** Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+      * , but each record is associated with the file path it comes from.
+      *
+      * Produces an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[(file_name, line)]</code>
+      * which provides a way to know from which file a given line comes from.
+      *
+      * {{{
+      * // Considering this folder:
+      * // folder/file_1.txt whose content is data1\ndata2\ndata3
+      * // folder/file_2.txt whose content is data4\ndata4
+      * // folder/folder_1/file_3.txt whose content is data6\ndata7
+      * // then:
+      * sc.textFileWithFileName("folder")
+      * // will return:
+      * RDD(
+      *   ("file:/path/on/machine/folder/file_1.txt", "data1"),
+      *   ("file:/path/on/machine/folder/file_1.txt", "data2"),
+      *   ("file:/path/on/machine/folder/file_1.txt", "data3"),
+      *   ("file:/path/on/machine/folder/file_2.txt", "data4"),
+      *   ("file:/path/on/machine/folder/file_2.txt", "data5"),
+      *   ("file:/path/on/machine/folder/folder_1/file_3.txt", "data6"),
+      *   ("file:/path/on/machine/folder/folder_1/file_3.txt", "data7")
+      * )
+      * }}}
+      *
+      * @param path the path of the folder (or structure of folders) to read
+      * @return the RDD of records where a record is a tuple containing the path
+      * of the file the record comes from and the record itself.
+      */
+    def textFileWithFileName(path: String): RDD[(String, String)] = {
+
+      // In order to go through the folder structure recursively:
+      sc.hadoopConfiguration
+        .set("mapreduce.input.fileinputformat.input.dir.recursive", "true")
+
+      sc.hadoopFile(
+          path,
+          classOf[TextInputFormat2],
+          classOf[LongWritable],
+          classOf[Text],
+          sc.defaultMinPartitions
+        )
+        .asInstanceOf[HadoopRDD[LongWritable, Text]]
+        .mapPartitionsWithInputSplit {
+          case (inputSplit, iterator) =>
+            val file = inputSplit.asInstanceOf[FileSplit]
+            iterator.map(tpl => (file.getPath.toString, tpl._2.toString))
+        }
+    }
+
     /** Decreases the nbr of partitions of a folder.
       *
       * This comes in handy when the last step of your job needs to run on
@@ -479,61 +529,6 @@ object SparkHelper extends Serializable {
       )
   }
 
-  /** Equivalent to sparkContext.textFile(), but for each line is associated
-    * with its file path.
-    *
-    * Produces a RDD[(file_name, line)] which provides a way to know from which
-    * file a given line comes from.
-    *
-    * {{{
-    * // Considering this folder:
-    * // folder/file_1.txt whose content is data1\ndata2\ndata3
-    * // folder/file_2.txt whose content is data4\ndata4
-    * // folder/folder_1/file_3.txt whose content is data6\ndata7
-    * // then:
-    * SparkHelper.textFileWithFileName("folder", sparkContext)
-    * // will return:
-    * RDD(
-    *   ("file:/path/on/machine/folder/file_1.txt", "data1"),
-    *   ("file:/path/on/machine/folder/file_1.txt", "data2"),
-    *   ("file:/path/on/machine/folder/file_1.txt", "data3"),
-    *   ("file:/path/on/machine/folder/file_2.txt", "data4"),
-    *   ("file:/path/on/machine/folder/file_2.txt", "data5"),
-    *   ("file:/path/on/machine/folder/folder_1/file_3.txt", "data6"),
-    *   ("file:/path/on/machine/folder/folder_1/file_3.txt", "data7")
-    * )
-    * }}}
-    *
-    * @param path the path of the folder (or structure of folders) to read
-    * @param sparkContext the SparkContext
-    * @return the RDD of records where a record is a tuple containing the path
-    * of the file the record comes from and the record itself.
-    */
-  def textFileWithFileName(
-      path: String,
-      sparkContext: SparkContext
-  ): RDD[(String, String)] = {
-
-    // In order to go through the folder structure recursively:
-    sparkContext.hadoopConfiguration
-      .set("mapreduce.input.fileinputformat.input.dir.recursive", "true")
-
-    sparkContext
-      .hadoopFile(
-        path,
-        classOf[TextInputFormat2],
-        classOf[LongWritable],
-        classOf[Text],
-        sparkContext.defaultMinPartitions
-      )
-      .asInstanceOf[HadoopRDD[LongWritable, Text]]
-      .mapPartitionsWithInputSplit {
-        case (inputSplit, iterator) =>
-          val file = inputSplit.asInstanceOf[FileSplit]
-          iterator.map(tpl => (file.getPath.toString, tpl._2.toString))
-      }
-  }
-
   // Internal core:
 
   private def saveAsSingleTextFileWithWorkingFolderInternal(
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 2af3b4d..a692350 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -312,19 +312,25 @@ class SparkHelperTest
     "Extract lines of files to an RDD of tuple containing the line and file " +
       "the line comes from") {
 
-    HdfsHelper.deleteFolder("src/test/resources/with_file_name")
+    val testFolder = s"$resourceFolder/with_file_name"
+
+    HdfsHelper.deleteFolder(testFolder)
+
     HdfsHelper.writeToHdfsFile(
       "data_1_a\ndata_1_b\ndata_1_c",
-      "src/test/resources/with_file_name/file_1.txt")
+      s"$testFolder/file_1.txt"
+    )
     HdfsHelper.writeToHdfsFile(
       "data_2_a\ndata_2_b",
-      "src/test/resources/with_file_name/file_2.txt")
+      s"$testFolder/file_2.txt"
+    )
     HdfsHelper.writeToHdfsFile(
       "data_3_a\ndata_3_b\ndata_3_c\ndata_3_d",
-      "src/test/resources/with_file_name/folder_1/file_3.txt")
+      s"$testFolder/folder_1/file_3.txt"
+    )
 
-    val computedRdd = SparkHelper
-      .textFileWithFileName("src/test/resources/with_file_name", sc)
+    val computedRdd = sc
+      .textFileWithFileName(testFolder)
       // We remove the part of the path which is specific to the local machine
       // on which the test run:
       .map {
@@ -359,6 +365,6 @@ class SparkHelperTest
 
     assertRDDEquals(computedRdd, expectedRDD)
 
-    HdfsHelper.deleteFolder("src/test/resources/with_file_name")
+    HdfsHelper.deleteFolder(testFolder)
   }
 }

From 9b04cbedf7309a8a869a25a75ba15457eedbf26e Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sat, 26 May 2018 12:42:41 +0100
Subject: [PATCH 09/25] Add code-style to doc

---
 .../scala/com/spark_helper/SparkHelper.scala  | 43 ++++++++++---------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 2ed582e..1da4b10 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -21,8 +21,8 @@ import scala.util.Random
   * A few exemples:
   *
   * {{{
-  * // Same as SparkContext.saveAsTextFile, but the result is a single file:
-  * SparkHelper.saveAsSingleTextFile(myOutputRDD, "/my/output/file/path.txt")
+  * // Same as sc.saveAsTextFile(path), but the result is a single file:
+  * rdd.saveAsSingleTextFile("/my/output/file/path.txt")
   * // Same as SparkContext.textFile, but instead of reading one record per
   * // line, it reads records spread over several lines.
   * // This way, xml, json, yml or any multi-line record file format can be used
@@ -78,10 +78,10 @@ object SparkHelper extends Serializable {
       * Allows one to save an RDD in one file, while keeping the processing
       * parallelized.
       *
-      * This variant of saveAsSingleTextFile performs the storage in a temporary
-      * folder instead of directly in the final output folder. This way the
-      * risks of having corrupted files in the real output folder due to cluster
-      * interruptions is minimized.
+      * This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
+      * performs the storage in a temporary folder instead of directly in the
+      * final output folder. This way the risks of having corrupted files in the
+      * real output folder due to cluster interruptions is minimized.
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path") }}}
       *
@@ -102,10 +102,10 @@ object SparkHelper extends Serializable {
       * Allows one to save an RDD in one file, while keeping the processing
       * parallelized.
       *
-      * This variant of saveAsSingleTextFile performs the storage in a temporary
-      * folder instead of directly in the final output folder. This way the risks
-      * of having corrupted files in the real output folder due to cluster
-      * interruptions is minimized.
+      * This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
+      * performs the storage in a temporary folder instead of directly in the
+      * final output folder. This way the risks of having corrupted files in the
+      * real output folder due to cluster interruptions is minimized.
       *
       * {{{
       * rdd.saveAsSingleTextFile("/my/file/path.txt", "/my/working/folder/path", classOf[BZip2Codec])
@@ -174,7 +174,7 @@ object SparkHelper extends Serializable {
       )
     }
 
-    /** Saves as text file, but by decreasing the nbr of partitions of the output.
+    /** Saves as text file, and decreases the nbr of output partitions.
       *
       * Same as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.saveAsTextFile()</code>
       * , but decreases the nbr of partitions in the output folder before doing
@@ -234,7 +234,7 @@ object SparkHelper extends Serializable {
       *
       * As this internally needs to know the nbr of keys, this will have to
       * compute it. If this nbr of keys is known beforehand, it would spare
-      * resources to use saveAsTextFileByKey(path: String, keyNbr: Int)
+      * resources to use <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsTextFileByKey(path: String, keyNbr: Int)</code>
       * instead.
       *
       * This is not scalable. This shouldn't be considered for any data flow
@@ -274,7 +274,7 @@ object SparkHelper extends Serializable {
       * As this internally needs to know the nbr of keys, this will have to
       * compute it. If this nbr of keys is known beforehand, it would spare
       * resources to use
-      * saveAsTextFileByKey(path: String, keyNbr: Int, codec: Class[_ <: CompressionCodec])
+      * <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsTextFileByKey(path: String, keyNbr: Int, codec: Class[_ <: CompressionCodec])</code>
       * instead.
       *
       * This is not scalable. This shouldn't be considered for any data flow
@@ -320,17 +320,20 @@ object SparkHelper extends Serializable {
 
   implicit class SparkContextExtensions(val sc: SparkContext) extends AnyVal {
 
-    /** Equivalent to sparkContext.textFile(), but for a specific record delimiter.
+    /** Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+      * , but for a specific record delimiter.
       *
-      * By default, sparkContext.textFile() will provide one record per line
-      * (per '\n'). But what if the format to read considers that one record
-      * is stored in more than one line (yml, custom format, ...)?
+      * By default, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+      * will provide one record per line (per <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">'\n'</code>).
+      * But what if the format to read considers that one record is stored in
+      * more than one line (yml, custom format, ...)?
       *
       * For instance in order to read a yml file, which is a format for which a
       * record (a single entity) is spread other several lines, you can modify
-      * the record delimiter with "---\n" instead of "\n". Same goes when
-      * reading an xml file where a record might be spread over several lines or
-      * worse the whole xml file is one line.
+      * the record delimiter with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">"---\n"</code>
+      * instead of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">"\n"</code>.
+      * Same goes when reading an xml file where a record might be spread over
+      * several lines or worse the whole xml file is one line.
       *
       * {{{
       * // Let's say data we want to use with Spark looks like this (one record

From 22cdff8ebfc93bf9b9467f5a625029c25ad067a8 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sat, 26 May 2018 15:27:22 +0100
Subject: [PATCH 10/25] Add a sc.textFile which reads files containing commas
 in their name

---
 .../scala/com/spark_helper/SparkHelper.scala  | 43 ++++++++++++++-
 .../org/apache/spark/TextFileOverwrite.scala  | 54 +++++++++++++++++++
 .../com/spark_helper/SparkHelperTest.scala    | 29 +++++++++-
 3 files changed, 122 insertions(+), 4 deletions(-)
 create mode 100644 src/main/scala/org/apache/spark/TextFileOverwrite.scala

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 1da4b10..73b5ccc 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -1,5 +1,7 @@
 package com.spark_helper
 
+import org.apache.spark.TextFileOverwrite
+
 import org.apache.spark.{HashPartitioner, SparkContext}
 import org.apache.spark.rdd.{RDD, HadoopRDD}
 import org.apache.hadoop.conf.Configuration
@@ -449,6 +451,43 @@ object SparkHelper extends Serializable {
         }
     }
 
+    /** A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+      * when files contains commas in their name.
+      *
+      * As <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+      * allows to provide several files at once by giving them as a string which
+      * is a list of strings joined with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">,</code>,
+      * we can't give it files containing commas in their name.
+      *
+      * This method aims at bypassing this limitation by passing paths as a
+      * sequence of strings.
+      *
+      * {{{ sc.textFile(Seq("path/hello,world.txt", "path/hello_world.txt")) }}}
+      *
+      * @param paths the paths of the file(s)/folder(s) to read
+      */
+    def textFile(paths: Seq[String]): RDD[String] =
+      TextFileOverwrite.textFile(paths, sc.defaultMinPartitions, sc)
+
+    /** A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+      * when files contains commas in their name.
+      *
+      * As <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+      * allows to provide several files at once by giving them as a string which
+      * is a list of strings joined with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">,</code>,
+      * we can't give it files containing commas in their name.
+      *
+      * This method aims at bypassing this limitation by passing paths as a
+      * sequence of strings.
+      *
+      * {{{ sc.textFile(Seq("path/hello,world.txt", "path/hello_world.txt")) }}}
+      *
+      * @param paths the paths of the file(s)/folder(s) to read
+      * @param minPartitions the nbr of partitions in which to split the input
+      */
+    def textFile(paths: Seq[String], minPartitions: Int): RDD[String] =
+      TextFileOverwrite.textFile(paths, minPartitions, sc)
+
     /** Decreases the nbr of partitions of a folder.
       *
       * This comes in handy when the last step of your job needs to run on
@@ -643,11 +682,11 @@ object SparkHelper extends Serializable {
       highCoalescenceLevelFolder: String,
       lowerCoalescenceLevelFolder: String,
       finalCoalesceLevel: Int,
-      sparkContext: SparkContext,
+      sc: SparkContext,
       codec: Option[Class[_ <: CompressionCodec]]
   ): Unit = {
 
-    val intermediateRDD = sparkContext
+    val intermediateRDD = sc
       .textFile(highCoalescenceLevelFolder)
       .coalesce(finalCoalesceLevel)
 
diff --git a/src/main/scala/org/apache/spark/TextFileOverwrite.scala b/src/main/scala/org/apache/spark/TextFileOverwrite.scala
new file mode 100644
index 0000000..8d1cc1b
--- /dev/null
+++ b/src/main/scala/org/apache/spark/TextFileOverwrite.scala
@@ -0,0 +1,54 @@
+package org.apache.spark
+
+import org.apache.spark.rdd.{RDD, HadoopRDD}
+import org.apache.spark.util.SerializableConfiguration
+import org.apache.hadoop.mapred.{FileInputFormat, JobConf, TextInputFormat}
+import org.apache.hadoop.io.{LongWritable, Text}
+import org.apache.hadoop.fs.Path
+
+object TextFileOverwrite {
+
+  def textFile(
+      paths: Seq[String],
+      minPartitions: Int,
+      sc: SparkContext
+  ): RDD[String] = {
+
+    /* Private notes:
+     *
+     * * Compared to sc.textFile(), the only difference in the implementation is
+     * the call to FileInputFormat.setInputPaths which takes Paths in input
+     * instead of a comma-separated String.
+     *
+     * * I use the package org.apache.spark to store this function, because
+     * SerializableConfiguration has the visibility private[spark] in spark's
+     * code base.
+     *
+     * * I would have preferred giving Seq[Path] instead of Seq[String] as an
+     * input of this method, but Path is not yet Serializable in the current
+     * version of hadoop-common used by Spark (it will become Serializable
+     * starting version 3 of hadoop-common).
+     *
+     * * I don't String* (instead of Seq[String]) as for 1 String only it would
+     * confuse the compiler as to which sc.textFile to use (the default one or
+     * this one).
+     */
+
+    val confBroadcast =
+      sc.broadcast(new SerializableConfiguration(sc.hadoopConfiguration))
+
+    val setInputPathsFunc =
+      (jobConf: JobConf) =>
+        FileInputFormat.setInputPaths(jobConf, paths.map(p => new Path(p)): _*)
+
+    new HadoopRDD(
+      sc,
+      confBroadcast,
+      Some(setInputPathsFunc),
+      classOf[TextInputFormat],
+      classOf[LongWritable],
+      classOf[Text],
+      minPartitions
+    ).map(pair => pair._2.toString)
+  }
+}
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index a692350..cbcea8d 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -342,7 +342,7 @@ class SparkHelperTest
           (nonLocalPath, line)
       }
 
-    val expectedRDD = sc.parallelize(
+    val expectedRdd = sc.parallelize(
       Array(
         ("file:/.../src/test/resources/with_file_name/file_1.txt", "data_1_a"),
         ("file:/.../src/test/resources/with_file_name/file_1.txt", "data_1_b"),
@@ -363,7 +363,32 @@ class SparkHelperTest
         ("file:/.../src/test/resources/with_file_name/file_2.txt", "data_2_b")
       ))
 
-    assertRDDEquals(computedRdd, expectedRDD)
+    assertRDDEquals(computedRdd, expectedRdd)
+
+    HdfsHelper.deleteFolder(testFolder)
+  }
+
+  test("textFile with files containing commas in their path") {
+
+    val testFolder = s"$resourceFolder/files_containing_commas"
+
+    HdfsHelper.deleteFolder(testFolder)
+
+    HdfsHelper.writeToHdfsFile(
+      "data_1_a\ndata_1_b",
+      s"$testFolder/file,1.txt"
+    )
+    HdfsHelper.writeToHdfsFile(
+      "data_2_a\ndata_2_b",
+      s"$testFolder/file_2.txt"
+    )
+
+    val computedRdd =
+      sc.textFile(List(s"$testFolder/file,1.txt", s"$testFolder/file_2.txt"))
+    val expectedRdd =
+      sc.parallelize("data_1_a\ndata_1_b\ndata_2_a\ndata_2_b".split("\n"))
+
+    assertRDDEquals(computedRdd, expectedRdd)
 
     HdfsHelper.deleteFolder(testFolder)
   }

From f4007de198726c853f66a73c65af2c69362aa4bc Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sun, 27 May 2018 09:07:38 +0100
Subject: [PATCH 11/25] Add as comment an alternative way of getting RDD with
 records' file name

---
 src/main/scala/com/spark_helper/SparkHelper.scala | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 73b5ccc..c8c1c9c 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -449,6 +449,18 @@ object SparkHelper extends Serializable {
             val file = inputSplit.asInstanceOf[FileSplit]
             iterator.map(tpl => (file.getPath.toString, tpl._2.toString))
         }
+
+      /* An other way of doing would be:
+       *
+       * import org.apache.spark.sql.functions.input_file_name
+       * import spark.implicits._
+       * 
+       * spark.read
+       *   .text(testFolder)
+       *   .select(input_file_name, $"value")
+       *   .as[(String, String)]
+       *   .rdd
+       */
     }
 
     /** A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>

From 815faaa849acc2f197ea0c195c7905141fae88da Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sun, 10 Jun 2018 09:57:08 +0100
Subject: [PATCH 12/25] Add todos

---
 src/main/scala/com/spark_helper/SparkHelper.scala | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index c8c1c9c..e762697 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -1,7 +1,6 @@
 package com.spark_helper
 
 import org.apache.spark.TextFileOverwrite
-
 import org.apache.spark.{HashPartitioner, SparkContext}
 import org.apache.spark.rdd.{RDD, HadoopRDD}
 import org.apache.hadoop.conf.Configuration
@@ -36,6 +35,17 @@ import scala.util.Random
   * SparkHelper.textFileWithFileName("folder", sparkContext)
   * }}}
   *
+  * @todo some kind of partialMap:
+  * 
+  * {{{
+  * RDD(1, 3, 2, 7, 8).partMap{ case a if a % 2 == 0 => 2 * a }
+  * res: RDD(1, 3, 4, 7, 16)
+  * in order to avoid:
+  * RDD(1, 3, 2, 7, 8).partMap{ case a if a % 2 == 0 => 2 * a; case a => a }
+  * }}}
+  *
+  * @todo sc.parallelize[T](elmts: T*) instead of sc.parallelize[T](elmts: Array[T])
+  *
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/SparkHelper.scala">SparkHelper</a>
   *

From 46edfee6164e4494962fe8f821eaeb68c16f1f60 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sun, 10 Jun 2018 10:20:43 +0100
Subject: [PATCH 13/25] Pimp RDDs with a flatten method

---
 .../scala/com/spark_helper/SparkHelper.scala  | 52 +++++++++++++++----
 .../com/spark_helper/SparkHelperTest.scala    | 16 ++++++
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index e762697..8718072 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -11,6 +11,8 @@ import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat
 import org.apache.hadoop.mapred.{FileSplit, TextInputFormat => TextInputFormat2}
 
+import scala.reflect.ClassTag
+
 import scala.util.Random
 
 /** A facility to deal with RDD/file manipulations based on the Spark API.
@@ -36,7 +38,7 @@ import scala.util.Random
   * }}}
   *
   * @todo some kind of partialMap:
-  * 
+  *
   * {{{
   * RDD(1, 3, 2, 7, 8).partMap{ case a if a % 2 == 0 => 2 * a }
   * res: RDD(1, 3, 4, 7, 16)
@@ -235,6 +237,34 @@ object SparkHelper extends Serializable {
     }
   }
 
+  implicit class SeqRDDExtensions[T: ClassTag](val rdd: RDD[Seq[T]]) {
+
+    /** Flattens an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[Seq[T]]</code>
+      * to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.
+      *
+      * {{{ sc.parallelize(Array(Seq(1, 2, 3), Nil, Seq(4))).flatten == sc.parallelize(Array(Seq(1, 2, 3, 4))) }}}
+      *
+      * @return the flat RDD as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD.flatMap(identity)</code>
+      * or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
+      * would have.
+      */
+    def flatten(): RDD[T] = rdd.flatMap(identity)
+  }
+
+  implicit class OptionRDDExtensions[T: ClassTag](val rdd: RDD[Option[T]]) {
+
+    /** Flattens an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[Option[T]]</code>
+      * to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.
+      *
+      * {{{ sc.parallelize(Array(Some(1), None, Some(2))).flatten == sc.parallelize(Array(Seq(1, 2))) }}}
+      *
+      * @return the flat RDD as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD.flatMap(x => x)</code>
+      * or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
+      * would have.
+      */
+    def flatten(): RDD[T] = rdd.flatMap(o => o)
+  }
+
   implicit class PairRDDExtensions(val rdd: RDD[(String, String)])
       extends AnyVal {
 
@@ -461,16 +491,16 @@ object SparkHelper extends Serializable {
         }
 
       /* An other way of doing would be:
-       *
-       * import org.apache.spark.sql.functions.input_file_name
-       * import spark.implicits._
-       * 
-       * spark.read
-       *   .text(testFolder)
-       *   .select(input_file_name, $"value")
-       *   .as[(String, String)]
-       *   .rdd
-       */
+     *
+     * import org.apache.spark.sql.functions.input_file_name
+     * import spark.implicits._
+     *
+     * spark.read
+     *   .text(testFolder)
+     *   .select(input_file_name, $"value")
+     *   .as[(String, String)]
+     *   .rdd
+     */
     }
 
     /** A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index cbcea8d..5f07fc1 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,6 +1,7 @@
 package com.spark_helper
 
 import com.spark_helper.SparkHelper.{RDDExtensions, PairRDDExtensions}
+import com.spark_helper.SparkHelper.{SeqRDDExtensions, OptionRDDExtensions}
 import com.spark_helper.SparkHelper.SparkContextExtensions
 
 import org.apache.hadoop.io.compress.GzipCodec
@@ -149,6 +150,21 @@ class SparkHelperTest
     HdfsHelper.deleteFile(xmlFilePath)
   }
 
+  test("Flatten RDD") {
+
+    var in = sc.parallelize(Array(Seq(1, 2, 3), Seq(), Nil, Seq(4), Seq(5, 6)))
+    var out = sc.parallelize(Array(1, 2, 3, 4, 5, 6))
+    assertRDDEquals(in.flatten, out)
+
+    in = sc.parallelize(Array(List(1, 2, 3), List(), Nil, List(4), List(5, 6)))
+    out = sc.parallelize(Array(1, 2, 3, 4, 5, 6))
+    assertRDDEquals(in.flatten, out)
+
+    val in2 = sc.parallelize(Array(Option(1), None, Option(2)))
+    val out2 = sc.parallelize(Array(1, 2))
+    assertRDDEquals(in2.flatten, out2)
+  }
+
   test("Save as text file by key") {
 
     val keyValueFolder = s"$resourceFolder/key_value_storage"

From 1e3a065195db8004cf1383fadc1ce10e1493e020 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Mon, 11 Jun 2018 23:23:26 +0100
Subject: [PATCH 14/25] Pimp RDDs with a partialMap function

---
 .../scala/com/spark_helper/SparkHelper.scala  | 46 ++++++++++++++-----
 .../com/spark_helper/SparkHelperTest.scala    | 12 ++++-
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 8718072..ff400f9 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -37,26 +37,48 @@ import scala.util.Random
   * SparkHelper.textFileWithFileName("folder", sparkContext)
   * }}}
   *
-  * @todo some kind of partialMap:
-  *
-  * {{{
-  * RDD(1, 3, 2, 7, 8).partMap{ case a if a % 2 == 0 => 2 * a }
-  * res: RDD(1, 3, 4, 7, 16)
-  * in order to avoid:
-  * RDD(1, 3, 2, 7, 8).partMap{ case a if a % 2 == 0 => 2 * a; case a => a }
-  * }}}
-  *
-  * @todo sc.parallelize[T](elmts: T*) instead of sc.parallelize[T](elmts: Array[T])
-  *
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/SparkHelper.scala">SparkHelper</a>
   *
+  * @todo sc.parallelize[T](elmts: T*) instead of sc.parallelize[T](elmts: Array[T])
   * @author Xavier Guihot
   * @since 2017-02
   */
 object SparkHelper extends Serializable {
 
-  implicit class RDDExtensions(val rdd: RDD[String]) extends AnyVal {
+  implicit class RDDExtensions[T: ClassTag](val rdd: RDD[T]) {
+
+    /** Map an RDD to the same type, by applying a partial function and the
+      * identity otherwise.
+      *
+      * Avoids having <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">case x => x</code>.
+      *
+      * Similar idea to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">.collect</code>,
+      * but instead of skipping non-matching items, keeps then as-is.
+      *
+      * {{{
+      * sc.parallelize(Array(1, 3, 2, 7, 8)).partialMap { case a if a % 2 == 0 => 2 * a }
+      * // is equivalent to:
+      * sc.parallelize(Array(1, 3, 2, 7, 8)).map {
+      *   case a if a % 2 == 0 => 2 * a
+      *   case a               => a
+      * }
+      * // in order to map to:
+      * sc.parallelize(Array(1, 3, 4, 7, 16))
+      * }}}
+      *
+      * @param pf the partial function to apply
+      * @return an rdd of the same type, for which each element is either the
+      * application of the partial function where defined or the identity.
+      */
+    def partialMap(pf: PartialFunction[T, T]): RDD[T] =
+      rdd.map {
+        case x if pf.isDefinedAt(x) => pf(x)
+        case x                      => x
+      }
+  }
+
+  implicit class StringRDDExtensions(val rdd: RDD[String]) extends AnyVal {
 
     /** Saves an RDD in exactly one file.
       *
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 5f07fc1..1ccfba4 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,8 +1,8 @@
 package com.spark_helper
 
-import com.spark_helper.SparkHelper.{RDDExtensions, PairRDDExtensions}
+import com.spark_helper.SparkHelper.{RDDExtensions, StringRDDExtensions}
 import com.spark_helper.SparkHelper.{SeqRDDExtensions, OptionRDDExtensions}
-import com.spark_helper.SparkHelper.SparkContextExtensions
+import com.spark_helper.SparkHelper.{SparkContextExtensions, PairRDDExtensions}
 
 import org.apache.hadoop.io.compress.GzipCodec
 
@@ -408,4 +408,12 @@ class SparkHelperTest
 
     HdfsHelper.deleteFolder(testFolder)
   }
+
+  test("Partial map") {
+
+    val in = sc.parallelize(Array(1, 3, 2, 7, 8))
+    val computedOut = in.partialMap { case a if a % 2 == 0 => 2 * a }
+    val expetcedOut = sc.parallelize(Array(1, 3, 4, 7, 16))
+    assertRDDEquals(computedOut, expetcedOut)
+  }
 }

From 277f22245938f4abe494df3f89dabd25cda758f6 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Mon, 11 Jun 2018 23:59:51 +0100
Subject: [PATCH 15/25] Add test for HdfsHelper.compressFile

---
 .../scala/com/spark_helper/HdfsHelper.scala   | 16 ++++++-------
 .../scala/com/spark_helper/SparkHelper.scala  |  2 +-
 .../com/spark_helper/HdfsHelperTest.scala     | 24 +++++++++++++++++++
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index f3dc3ed..a9038bd 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -340,8 +340,7 @@ object HdfsHelper extends Serializable {
     FileSystem
       .get(new Configuration())
       .listStatus(new Path(hdfsPath))
-      .flatMap(status => {
-
+      .flatMap { status =>
         // If it's a file:
         if (status.isFile) {
           if (onlyName) List(status.getPath.getName)
@@ -352,11 +351,12 @@ object HdfsHelper extends Serializable {
           listFileNamesInFolder(
             hdfsPath + "/" + status.getPath.getName,
             true,
-            onlyName)
+            onlyName
+          )
         // If it's a dir and we're not in a recursive option:
         else
           Nil
-      })
+      }
       .toList
       .sorted
   }
@@ -661,8 +661,8 @@ object HdfsHelper extends Serializable {
     val ClassOfBZip2 = classOf[BZip2Codec]
 
     val outputPath = compressionCodec match {
-      case ClassOfGzip  => inputPath + ".gz"
-      case ClassOfBZip2 => inputPath + ".bz2"
+      case ClassOfGzip  => s"$inputPath.gz"
+      case ClassOfBZip2 => s"$inputPath.bz2"
     }
 
     val inputStream = fileSystem.open(new Path(inputPath))
@@ -747,8 +747,8 @@ object HdfsHelper extends Serializable {
     val fileSystem = FileSystem.get(new Configuration())
 
     val tmpOutputPath = workingFolderPath match {
-      case "" => filePath + ".tmp"
-      case _  => workingFolderPath + "/xml.tmp"
+      case "" => s"$filePath.tmp"
+      case _  => s"$workingFolderPath/xml.tmp"
     }
     deleteFile(tmpOutputPath)
 
diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index ff400f9..075c22f 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -54,7 +54,7 @@ object SparkHelper extends Serializable {
       * Avoids having <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">case x => x</code>.
       *
       * Similar idea to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">.collect</code>,
-      * but instead of skipping non-matching items, keeps then as-is.
+      * but instead of skipping non-matching items, it keeps them as-is.
       *
       * {{{
       * sc.parallelize(Array(1, 3, 2, 7, 8)).partialMap { case a if a % 2 == 0 => 2 * a }
diff --git a/src/test/scala/com/spark_helper/HdfsHelperTest.scala b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
index 1935e89..8c807d9 100644
--- a/src/test/scala/com/spark_helper/HdfsHelperTest.scala
+++ b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
@@ -1,5 +1,7 @@
 package com.spark_helper
 
+import org.apache.hadoop.io.compress.GzipCodec
+
 import com.holdenkarau.spark.testing.SharedSparkContext
 
 import org.scalatest.FunSuite
@@ -11,6 +13,8 @@ import org.scalatest.FunSuite
   */
 class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
+  val resourceFolder = "src/test/resources"
+
   test("Delete file/folder") {
 
     // Let's try to delete a file:
@@ -538,4 +542,24 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
     HdfsHelper.deleteFolder("src/test/resources/folder_to_purge")
   }
+
+  test("Compress hdfs file") {
+
+    val testFolder = s"$resourceFolder/folder"
+    val filePath = s"$testFolder/file.txt"
+
+    HdfsHelper.deleteFile(filePath)
+
+    HdfsHelper.writeToHdfsFile("hello\nworld", filePath)
+    HdfsHelper.compressFile(filePath, classOf[GzipCodec], true)
+
+    assert(HdfsHelper.fileExists(s"$filePath.gz"))
+
+    // Easy to test with spark, as reading a file with the ".gz" extention
+    // forces the read with the compression codec:
+    val content = sc.textFile(s"$filePath.gz").collect.sorted
+    assert(content === Array("hello", "world"))
+
+    HdfsHelper.deleteFolder(testFolder)
+  }
 }

From f108bacfba679d1271ae40851da7ef8d93cbfe2f Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Tue, 12 Jun 2018 22:10:12 +0100
Subject: [PATCH 16/25] Clean-up of HdfsHelperTest

---
 .../com/spark_helper/HdfsHelperTest.scala     | 393 +++++++-----------
 1 file changed, 161 insertions(+), 232 deletions(-)

diff --git a/src/test/scala/com/spark_helper/HdfsHelperTest.scala b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
index 8c807d9..9babb0b 100644
--- a/src/test/scala/com/spark_helper/HdfsHelperTest.scala
+++ b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
@@ -14,61 +14,66 @@ import org.scalatest.FunSuite
 class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
   val resourceFolder = "src/test/resources"
+  val testFolder = s"$resourceFolder/folder"
 
   test("Delete file/folder") {
 
+    val filePath = s"$testFolder/file.txt"
+
     // Let's try to delete a file:
 
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/file_to_delete.txt")
+    HdfsHelper.createEmptyHdfsFile(filePath)
 
     // 1: Let's try to delete it with the deleteFolder method:
     var messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.deleteFolder("src/test/resources/file_to_delete.txt")
+      HdfsHelper.deleteFolder(filePath)
     }
     var expectedMessage =
       "requirement failed: to delete a file, prefer using the " +
         "deleteFile() method."
     assert(messageThrown.getMessage === expectedMessage)
-    assert(HdfsHelper.fileExists("src/test/resources/file_to_delete.txt"))
+    assert(HdfsHelper.fileExists(filePath))
 
     // 2: Let's delete it with the deleteFile method:
-    HdfsHelper.deleteFile("src/test/resources/file_to_delete.txt")
-    assert(!HdfsHelper.fileExists("src/test/resources/file_to_delete.txt"))
+    HdfsHelper.deleteFile(filePath)
+    assert(!HdfsHelper.fileExists(filePath))
 
     // Let's try to delete a folder:
 
-    HdfsHelper
-      .writeToHdfsFile("", "src/test/resources/folder_to_delete/file.txt")
+    HdfsHelper.createEmptyHdfsFile(s"$testFolder/file.txt")
 
     // 3: Let's try to delete it with the deleteFile method:
     messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.deleteFile("src/test/resources/folder_to_delete")
+      HdfsHelper.deleteFile(testFolder)
     }
     expectedMessage =
       "requirement failed: to delete a folder, prefer using the " +
         "deleteFolder() method."
     assert(messageThrown.getMessage === expectedMessage)
-    assert(HdfsHelper.folderExists("src/test/resources/folder_to_delete"))
+    assert(HdfsHelper.folderExists(testFolder))
 
     // 4: Let's delete it with the deleteFolder method:
-    HdfsHelper.deleteFolder("src/test/resources/folder_to_delete")
-    assert(!HdfsHelper.folderExists("src/test/resources/folder_to_delete"))
+    HdfsHelper.deleteFolder(testFolder)
+    assert(!HdfsHelper.folderExists(testFolder))
   }
 
   test("File/folder exists") {
 
-    HdfsHelper.deleteFile("src/test/resources/file_to_check.txt")
-    HdfsHelper.deleteFolder("src/test/resources/folder_to_check")
+    val folderPath = s"$resourceFolder/folder"
+    val filePath = s"$folderPath/file.txt"
+
+    HdfsHelper.deleteFile(filePath)
+    HdfsHelper.deleteFolder(folderPath)
 
     // Let's try to check if a file exists:
 
-    assert(!HdfsHelper.fileExists("src/test/resources/file_to_check.txt"))
+    assert(!HdfsHelper.fileExists(filePath))
 
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/file_to_check.txt")
+    HdfsHelper.createEmptyHdfsFile(filePath)
 
     // 1: Let's try to check it exists with the folderExists method:
     var messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.folderExists("src/test/resources/file_to_check.txt")
+      HdfsHelper.folderExists(filePath)
     }
     var expectedMessage =
       "requirement failed: to check if a file exists, prefer using the " +
@@ -76,18 +81,18 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     assert(messageThrown.getMessage === expectedMessage)
 
     // 2: Let's try to check it exists with the fileExists method:
-    assert(HdfsHelper.fileExists("src/test/resources/file_to_check.txt"))
+    assert(HdfsHelper.fileExists(filePath))
 
     // Let's try to check if a folder exists:
 
-    assert(!HdfsHelper.folderExists("src/test/resources/folder_to_check"))
+    HdfsHelper.deleteFolder(folderPath)
+    assert(!HdfsHelper.folderExists(folderPath))
 
-    HdfsHelper
-      .writeToHdfsFile("", "src/test/resources/folder_to_check/file.txt")
+    HdfsHelper.createEmptyHdfsFile(filePath)
 
     // 3: Let's try to check it exists with the fileExists method:
     messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.fileExists("src/test/resources/folder_to_check")
+      HdfsHelper.fileExists(folderPath)
     }
     expectedMessage =
       "requirement failed: to check if a folder exists, prefer using " +
@@ -95,377 +100,306 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     assert(messageThrown.getMessage === expectedMessage)
 
     // 2: Let's try to check it exists with the folderExists method:
-    assert(HdfsHelper.folderExists("src/test/resources/folder_to_check"))
+    assert(HdfsHelper.folderExists(folderPath))
 
-    HdfsHelper.deleteFile("src/test/resources/file_to_check.txt")
-    HdfsHelper.deleteFolder("src/test/resources/folder_to_check")
+    HdfsHelper.deleteFile(filePath)
+    HdfsHelper.deleteFolder(folderPath)
   }
 
   test("Create an empty file on hdfs") {
 
-    HdfsHelper.deleteFile("src/test/resources/empty_file.token")
+    val filePath = s"$testFolder/empty_file.token"
 
-    HdfsHelper.createEmptyHdfsFile("src/test/resources/empty_file.token")
+    HdfsHelper.deleteFile(filePath)
 
-    assert(HdfsHelper.fileExists("src/test/resources/empty_file.token"))
+    HdfsHelper.createEmptyHdfsFile(filePath)
 
-    val tokenContent = sc
-      .textFile("src/test/resources/empty_file.token")
-      .collect()
-      .sorted
-      .mkString("\n")
+    assert(HdfsHelper.fileExists(filePath))
 
+    val tokenContent = sc.textFile(filePath).collect().sorted.mkString("\n")
     assert(tokenContent === "")
 
-    HdfsHelper.deleteFile("src/test/resources/empty_file.token")
+    HdfsHelper.deleteFile(filePath)
   }
 
   test(
     "Save text in HDFS file with the fileSystem API instead of the Spark API") {
 
+    val filePath = s"$testFolder/small_file.txt"
+
     // 1: Stores using a "\n"-joined string:
 
-    HdfsHelper.deleteFile("src/test/resources/folder/small_file.txt")
+    HdfsHelper.deleteFile(filePath)
 
     val contentToStore = "Hello World\nWhatever"
 
-    HdfsHelper.writeToHdfsFile(
-      contentToStore,
-      "src/test/resources/folder/small_file.txt")
+    HdfsHelper.writeToHdfsFile(contentToStore, filePath)
 
-    assert(HdfsHelper.fileExists("src/test/resources/folder/small_file.txt"))
-
-    var storedContent = sc
-      .textFile("src/test/resources/folder/small_file.txt")
-      .collect()
-      .sorted
-      .mkString("\n")
+    assert(HdfsHelper.fileExists(filePath))
 
+    var storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
     assert(storedContent === contentToStore)
 
-    HdfsHelper.deleteFolder("src/test/resources/folder")
+    HdfsHelper.deleteFolder(testFolder)
 
     // 2: Stores using a list of strings to be "\n"-joined:
 
-    HdfsHelper.deleteFile("src/test/resources/folder/small_file.txt")
+    HdfsHelper.deleteFile(filePath)
 
     val listToStore = List("Hello World", "Whatever")
+    HdfsHelper.writeToHdfsFile(listToStore, filePath)
 
-    HdfsHelper
-      .writeToHdfsFile(listToStore, "src/test/resources/folder/small_file.txt")
-
-    assert(HdfsHelper.fileExists("src/test/resources/folder/small_file.txt"))
-
-    storedContent = sc
-      .textFile("src/test/resources/folder/small_file.txt")
-      .collect()
-      .sorted
-      .mkString("\n")
+    assert(HdfsHelper.fileExists(filePath))
 
+    storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
     assert(storedContent === listToStore.mkString("\n"))
 
-    HdfsHelper.deleteFolder("src/test/resources/folder")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("List file names in Hdfs folder") {
 
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/folder_1/file_1.txt")
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/folder_1/file_2.csv")
-    HdfsHelper
-      .writeToHdfsFile("", "src/test/resources/folder_1/folder_2/file_3.txt")
+    val folder1 = s"$resourceFolder/folder_1"
+
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/file_1.txt")
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/file_2.csv")
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/folder_2/file_3.txt")
 
     // 1: Not recursive, names only:
-    var fileNames =
-      HdfsHelper.listFileNamesInFolder("src/test/resources/folder_1")
+    var fileNames = HdfsHelper.listFileNamesInFolder(folder1)
     var expectedFileNames = List("file_1.txt", "file_2.csv")
     assert(fileNames === expectedFileNames)
 
     // 2: Not recursive, full paths:
-    fileNames = HdfsHelper
-      .listFileNamesInFolder("src/test/resources/folder_1", onlyName = false)
-    expectedFileNames = List(
-      "src/test/resources/folder_1/file_1.txt",
-      "src/test/resources/folder_1/file_2.csv"
-    )
+    fileNames = HdfsHelper.listFileNamesInFolder(folder1, onlyName = false)
+    expectedFileNames = List(s"$folder1/file_1.txt", s"$folder1/file_2.csv")
     assert(fileNames === expectedFileNames)
 
     // 3: Recursive, names only:
-    fileNames = HdfsHelper
-      .listFileNamesInFolder("src/test/resources/folder_1", recursive = true)
+    fileNames = HdfsHelper.listFileNamesInFolder(folder1, recursive = true)
     expectedFileNames = List("file_1.txt", "file_2.csv", "file_3.txt")
     assert(fileNames === expectedFileNames)
 
     // 4: Recursive, full paths:
-    fileNames = HdfsHelper.listFileNamesInFolder(
-      "src/test/resources/folder_1",
-      recursive = true,
-      onlyName = false)
+    fileNames = HdfsHelper
+      .listFileNamesInFolder(folder1, recursive = true, onlyName = false)
     expectedFileNames = List(
-      "src/test/resources/folder_1/file_1.txt",
-      "src/test/resources/folder_1/file_2.csv",
-      "src/test/resources/folder_1/folder_2/file_3.txt"
+      s"$folder1/file_1.txt",
+      s"$folder1/file_2.csv",
+      s"$folder1/folder_2/file_3.txt"
     )
     assert(fileNames === expectedFileNames)
 
-    HdfsHelper.deleteFolder("src/test/resources/folder_1")
+    HdfsHelper.deleteFolder(folder1)
   }
 
   test("List folder names in Hdfs folder") {
 
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/folder_1/file_1.txt")
-    HdfsHelper
-      .writeToHdfsFile("", "src/test/resources/folder_1/folder_2/file_2.txt")
-    HdfsHelper
-      .writeToHdfsFile("", "src/test/resources/folder_1/folder_3/file_3.txt")
+    val folder1 = s"$resourceFolder/folder_1"
 
-    val folderNames = HdfsHelper.listFolderNamesInFolder(
-      "src/test/resources/folder_1"
-    )
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/file_1.txt")
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/folder_2/file_2.txt")
+    HdfsHelper.createEmptyHdfsFile(s"$folder1/folder_3/file_3.txt")
+
+    val folderNames = HdfsHelper.listFolderNamesInFolder(folder1)
     val expectedFolderNames = List("folder_2", "folder_3")
 
     assert(folderNames === expectedFolderNames)
 
-    HdfsHelper.deleteFolder("src/test/resources/folder_1")
+    HdfsHelper.deleteFolder(folder1)
   }
 
   test("Move file") {
 
+    val filePath = s"$testFolder/some_file.txt"
+    val renamedPath = s"$testFolder/renamed_file.txt"
+
     // Let's remove possible previous stuff:
-    HdfsHelper.deleteFile("src/test/resources/some_file.txt")
-    HdfsHelper.deleteFile("src/test/resources/renamed_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
 
     // Let's create the file to rename:
-    HdfsHelper.writeToHdfsFile("whatever", "src/test/resources/some_file.txt")
+    HdfsHelper.writeToHdfsFile("whatever", filePath)
 
     // 1: Let's try to move the file on a file which already exists without
     // the overwrite option:
 
-    assert(HdfsHelper.fileExists("src/test/resources/some_file.txt"))
-    assert(!HdfsHelper.fileExists("src/test/resources/renamed_file.txt"))
+    assert(HdfsHelper.fileExists(filePath))
+    assert(!HdfsHelper.fileExists(renamedPath))
 
     // Let's create the existing file where we want to move our file:
-    HdfsHelper.writeToHdfsFile("", "src/test/resources/renamed_file.txt")
+    HdfsHelper.createEmptyHdfsFile(renamedPath)
 
     // Let's rename the file to the path where a file already exists:
     val ioExceptionThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.moveFile(
-        "src/test/resources/some_file.txt",
-        "src/test/resources/renamed_file.txt")
+      HdfsHelper.moveFile(filePath, renamedPath)
     }
     var expectedMessage =
       "requirement failed: overwrite option set to false, but a file " +
-        "already exists at target location src/test/resources/renamed_file.txt"
+        "already exists at target location " +
+        "src/test/resources/folder/renamed_file.txt"
     assert(ioExceptionThrown.getMessage === expectedMessage)
 
-    assert(HdfsHelper.fileExists("src/test/resources/some_file.txt"))
-    assert(HdfsHelper.fileExists("src/test/resources/renamed_file.txt"))
+    assert(HdfsHelper.fileExists(filePath))
+    assert(HdfsHelper.fileExists(renamedPath))
 
-    HdfsHelper.deleteFile("src/test/resources/renamed_file.txt")
+    HdfsHelper.deleteFile(renamedPath)
 
     // 2: Let's fail to move the file with the moveFolder() method:
 
-    assert(HdfsHelper.fileExists("src/test/resources/some_file.txt"))
-    assert(!HdfsHelper.fileExists("src/test/resources/renamed_file.txt"))
+    assert(HdfsHelper.fileExists(filePath))
+    assert(!HdfsHelper.fileExists(renamedPath))
 
     // Let's rename the file:
     val illegalArgExceptionThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.moveFolder(
-        "src/test/resources/some_file.txt",
-        "src/test/resources/renamed_file.txt")
+      HdfsHelper.moveFolder(filePath, renamedPath)
     }
     expectedMessage =
       "requirement failed: to move a file, prefer using the " +
         "moveFile() method."
     assert(illegalArgExceptionThrown.getMessage === expectedMessage)
 
-    assert(HdfsHelper.fileExists("src/test/resources/some_file.txt"))
-    assert(!HdfsHelper.fileExists("src/test/resources/renamed_file.txt"))
+    assert(HdfsHelper.fileExists(filePath))
+    assert(!HdfsHelper.fileExists(renamedPath))
 
     // 3: Let's successfuly move the file with the moveFile() method:
 
     // Let's rename the file:
-    HdfsHelper.moveFile(
-      "src/test/resources/some_file.txt",
-      "src/test/resources/renamed_file.txt")
-
-    assert(!HdfsHelper.fileExists("src/test/resources/some_file.txt"))
-    assert(HdfsHelper.fileExists("src/test/resources/renamed_file.txt"))
+    HdfsHelper.moveFile(filePath, renamedPath)
 
-    val newContent = sc.textFile("src/test/resources/renamed_file.txt").collect
+    assert(!HdfsHelper.fileExists(filePath))
+    assert(HdfsHelper.fileExists(renamedPath))
 
+    val newContent = sc.textFile(renamedPath).collect
     assert(Array("whatever") === newContent)
 
-    HdfsHelper.deleteFile("src/test/resources/renamed_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Move folder") {
 
+    val folderToMove = s"$testFolder/folder_to_move"
+    val renamedFolder = s"$testFolder/renamed_folder"
+
     // Let's remove possible previous stuff:
-    HdfsHelper.deleteFolder("src/test/resources/some_folder_to_move")
-    HdfsHelper.deleteFolder("src/test/resources/renamed_folder")
+    HdfsHelper.deleteFolder(testFolder)
 
     // Let's create the folder to rename:
-    HdfsHelper.writeToHdfsFile(
-      "whatever",
-      "src/test/resources/some_folder_to_move/file_1.txt")
-    HdfsHelper.writeToHdfsFile(
-      "something",
-      "src/test/resources/some_folder_to_move/file_2.txt")
+    HdfsHelper.writeToHdfsFile("whatever", s"$folderToMove/file_1.txt")
+    HdfsHelper.writeToHdfsFile("something", s"$folderToMove/file_2.txt")
 
     // 1: Let's fail to move the folder with the moveFile() method:
 
-    assert(
-      HdfsHelper.fileExists(
-        "src/test/resources/some_folder_to_move/file_1.txt"))
-    assert(
-      HdfsHelper.fileExists(
-        "src/test/resources/some_folder_to_move/file_2.txt"))
-    assert(!HdfsHelper.folderExists("src/test/resources/renamed_folder"))
+    assert(HdfsHelper.fileExists(s"$folderToMove/file_1.txt"))
+    assert(HdfsHelper.fileExists(s"$folderToMove/file_2.txt"))
+    assert(!HdfsHelper.folderExists(renamedFolder))
 
     // Let's rename the folder:
     val messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.moveFile(
-        "src/test/resources/some_folder_to_move",
-        "src/test/resources/renamed_folder")
+      HdfsHelper.moveFile(folderToMove, renamedFolder)
     }
     val expectedMessage =
       "requirement failed: to move a folder, prefer using the " +
         "moveFolder() method."
     assert(messageThrown.getMessage === expectedMessage)
 
-    assert(
-      HdfsHelper.fileExists(
-        "src/test/resources/some_folder_to_move/file_1.txt"))
-    assert(
-      HdfsHelper.fileExists(
-        "src/test/resources/some_folder_to_move/file_2.txt"))
-    assert(!HdfsHelper.folderExists("src/test/resources/renamed_folder"))
+    assert(HdfsHelper.fileExists(s"$folderToMove/file_1.txt"))
+    assert(HdfsHelper.fileExists(s"$folderToMove/file_2.txt"))
+    assert(!HdfsHelper.folderExists(renamedFolder))
 
     // 2: Let's successfuly move the folder with the moveFolder() method:
 
     // Let's rename the folder:
-    HdfsHelper.moveFolder(
-      "src/test/resources/some_folder_to_move",
-      "src/test/resources/renamed_folder")
+    HdfsHelper.moveFolder(folderToMove, renamedFolder)
 
-    assert(!HdfsHelper.folderExists("src/test/resources/some_folder_to_move"))
-    assert(
-      HdfsHelper.fileExists("src/test/resources/renamed_folder/file_1.txt"))
-    assert(
-      HdfsHelper.fileExists("src/test/resources/renamed_folder/file_2.txt"))
-
-    val newContent =
-      sc.textFile("src/test/resources/renamed_folder").collect().sorted
+    assert(!HdfsHelper.folderExists(folderToMove))
+    assert(HdfsHelper.fileExists(s"$renamedFolder/file_1.txt"))
+    assert(HdfsHelper.fileExists(s"$renamedFolder/file_2.txt"))
 
+    val newContent = sc.textFile(renamedFolder).collect().sorted
     assert(newContent === Array("something", "whatever"))
 
-    HdfsHelper.deleteFolder("src/test/resources/renamed_folder")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Append header and footer to file") {
 
+    val filePath = s"$testFolder/header_footer_file.txt"
+    val tmpFolder = s"$testFolder/header_footer_tmp"
+
     // 1: Without the tmp/working folder:
 
-    HdfsHelper.deleteFile("src/test/resources/header_footer_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
 
     // Let's create the file for which to add header and footer:
-    HdfsHelper.writeToHdfsFile(
-      "whatever\nsomething else\n",
-      "src/test/resources/header_footer_file.txt")
+    HdfsHelper.writeToHdfsFile("whatever\nsomething else\n", filePath)
 
-    HdfsHelper.appendHeaderAndFooter(
-      "src/test/resources/header_footer_file.txt",
-      "my_header",
-      "my_footer")
+    HdfsHelper.appendHeaderAndFooter(filePath, "my_header", "my_footer")
 
-    var newContent = sc
-      .textFile("src/test/resources/header_footer_file.txt")
-      .collect
-      .mkString("\n")
+    var newContent = sc.textFile(filePath).collect.mkString("\n")
 
-    var expectedNewContent = (
+    var expectedNewContent =
       "my_header\n" +
         "whatever\n" +
         "something else\n" +
         "my_footer"
-    )
 
     assert(newContent === expectedNewContent)
 
-    HdfsHelper.deleteFile("src/test/resources/header_footer_file.txt")
+    HdfsHelper.deleteFile(filePath)
 
     // 2: With the tmp/working folder:
 
     // Let's create the file for which to add header and footer:
-    HdfsHelper.writeToHdfsFile(
-      "whatever\nsomething else\n",
-      "src/test/resources/header_footer_file.txt")
+    HdfsHelper.writeToHdfsFile("whatever\nsomething else\n", filePath)
 
-    HdfsHelper.appendHeaderAndFooter(
-      "src/test/resources/header_footer_file.txt",
-      "my_header",
-      "my_footer",
-      workingFolderPath = "src/test/resources/header_footer_tmp")
+    HdfsHelper
+      .appendHeaderAndFooter(filePath, "my_header", "my_footer", tmpFolder)
 
-    assert(HdfsHelper.folderExists("src/test/resources/header_footer_tmp"))
-    assert(
-      !HdfsHelper.fileExists("src/test/resources/header_footer_tmp/xml.tmp"))
+    assert(HdfsHelper.folderExists(tmpFolder))
+    assert(!HdfsHelper.fileExists(s"$tmpFolder/xml.tmp"))
 
-    newContent = sc
-      .textFile("src/test/resources/header_footer_file.txt")
-      .collect
-      .mkString("\n")
+    newContent = sc.textFile(filePath).collect.mkString("\n")
 
-    expectedNewContent = (
+    expectedNewContent =
       "my_header\n" +
         "whatever\n" +
         "something else\n" +
         "my_footer"
-    )
 
     assert(newContent === expectedNewContent)
 
-    HdfsHelper.deleteFile("src/test/resources/header_footer_file.txt")
-    HdfsHelper.deleteFolder("src/test/resources/header_footer_tmp")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Validate Xml Hdfs file with Xsd") {
 
+    val xmlPath = s"$testFolder/file.xml"
+
     // 1: Valid xml:
-    HdfsHelper.deleteFile("src/test/resources/xml_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
     HdfsHelper.writeToHdfsFile(
       "<Customer>\n" +
         "	<Age>24</Age>\n" +
         "	<Address>34 thingy street, someplace, sometown</Address>\n" +
         "</Customer>",
-      "src/test/resources/xml_file.txt"
+      xmlPath
     )
-
     var xsdFile = getClass.getResource("/some_xml.xsd")
-
-    var isValid = HdfsHelper
-      .isHdfsXmlCompliantWithXsd("src/test/resources/xml_file.txt", xsdFile)
-
-    assert(isValid)
+    assert(HdfsHelper.isHdfsXmlCompliantWithXsd(xmlPath, xsdFile))
 
     // 2: Invalid xml:
-    HdfsHelper.deleteFile("src/test/resources/xml_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
     HdfsHelper.writeToHdfsFile(
       "<Customer>\n" +
         "	<Age>trente</Age>\n" +
         "	<Address>34 thingy street, someplace, sometown</Address>\n" +
         "</Customer>",
-      "src/test/resources/xml_file.txt"
+      xmlPath
     )
-
     xsdFile = getClass.getResource("/some_xml.xsd")
+    assert(!HdfsHelper.isHdfsXmlCompliantWithXsd(xmlPath, xsdFile))
 
-    isValid = HdfsHelper
-      .isHdfsXmlCompliantWithXsd("src/test/resources/xml_file.txt", xsdFile)
-
-    assert(!isValid)
-
-    HdfsHelper.deleteFile("src/test/resources/xml_file.txt")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Load Typesafe Config from Hdfs") {
@@ -488,64 +422,59 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
   test("Load Xml file from Hdfs") {
 
-    HdfsHelper.deleteFile("src/test/resources/folder/xml_to_load.xml")
+    val xmlPath = s"$testFolder/file.xml"
+
+    HdfsHelper.deleteFolder(testFolder)
 
     HdfsHelper.writeToHdfsFile(
       "<toptag>\n" +
         "	<sometag value=\"something\">whatever</sometag>\n" +
         "</toptag>",
-      "src/test/resources/folder/xml_to_load.xml"
+      xmlPath
     )
 
-    val xmlContent = HdfsHelper
-      .loadXmlFileFromHdfs("src/test/resources/folder/xml_to_load.xml")
+    val xmlContent = HdfsHelper.loadXmlFileFromHdfs(xmlPath)
 
     assert((xmlContent \ "sometag" \ "@value").text === "something")
     assert((xmlContent \ "sometag").text === "whatever")
 
-    HdfsHelper.deleteFolder("src/test/resources/folder/")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Purge folder from too old files/folders") {
 
-    HdfsHelper.deleteFolder("src/test/resources/folder_to_purge")
-    HdfsHelper
-      .createEmptyHdfsFile("src/test/resources/folder_to_purge/file.txt")
-    HdfsHelper
-      .createEmptyHdfsFile("src/test/resources/folder_to_purge/folder/file.txt")
-    assert(HdfsHelper.fileExists("src/test/resources/folder_to_purge/file.txt"))
-    assert(HdfsHelper.folderExists("src/test/resources/folder_to_purge/folder"))
-
-    HdfsHelper.purgeFolder("src/test/resources/folder_to_purge", 63)
+    val folderToPurge = s"$testFolder/folder_to_purge"
 
-    assert(HdfsHelper.fileExists("src/test/resources/folder_to_purge/file.txt"))
-    assert(HdfsHelper.folderExists("src/test/resources/folder_to_purge/folder"))
+    HdfsHelper.deleteFolder(testFolder)
+    HdfsHelper.createEmptyHdfsFile(s"$folderToPurge/file.txt")
+    HdfsHelper.createEmptyHdfsFile(s"$folderToPurge/folder/file.txt")
+    assert(HdfsHelper.fileExists(s"$folderToPurge/file.txt"))
+    assert(HdfsHelper.folderExists(s"$folderToPurge/folder"))
 
-    HdfsHelper.purgeFolder("src/test/resources/folder_to_purge", 1)
+    HdfsHelper.purgeFolder(folderToPurge, 63)
+    assert(HdfsHelper.fileExists(s"$folderToPurge/file.txt"))
+    assert(HdfsHelper.folderExists(s"$folderToPurge/folder"))
 
-    assert(HdfsHelper.fileExists("src/test/resources/folder_to_purge/file.txt"))
-    assert(HdfsHelper.folderExists("src/test/resources/folder_to_purge/folder"))
+    HdfsHelper.purgeFolder(folderToPurge, 1)
+    assert(HdfsHelper.fileExists(s"$folderToPurge/file.txt"))
+    assert(HdfsHelper.folderExists(s"$folderToPurge/folder"))
 
     val messageThrown = intercept[IllegalArgumentException] {
-      HdfsHelper.purgeFolder("src/test/resources/folder_to_purge", -3)
+      HdfsHelper.purgeFolder(folderToPurge, -3)
     }
     val expectedMessage =
       "requirement failed: the purgeAge provided \"-3\" must be superior to 0."
     assert(messageThrown.getMessage === expectedMessage)
 
-    HdfsHelper.purgeFolder("src/test/resources/folder_to_purge", 0)
+    HdfsHelper.purgeFolder(folderToPurge, 0)
+    assert(!HdfsHelper.fileExists(s"$folderToPurge/file.txt"))
+    assert(!HdfsHelper.folderExists(s"$folderToPurge/folder"))
 
-    assert(
-      !HdfsHelper.fileExists("src/test/resources/folder_to_purge/file.txt"))
-    assert(
-      !HdfsHelper.folderExists("src/test/resources/folder_to_purge/folder"))
-
-    HdfsHelper.deleteFolder("src/test/resources/folder_to_purge")
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("Compress hdfs file") {
 
-    val testFolder = s"$resourceFolder/folder"
     val filePath = s"$testFolder/file.txt"
 
     HdfsHelper.deleteFile(filePath)

From 21bbb8041e9f381a536d916f1bae258c8c0f8453 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Tue, 12 Jun 2018 22:22:56 +0100
Subject: [PATCH 17/25] Light refactoring of DateHelperTest

---
 .../com/spark_helper/DateHelperTest.scala     | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/test/scala/com/spark_helper/DateHelperTest.scala b/src/test/scala/com/spark_helper/DateHelperTest.scala
index 7154831..6d27b81 100644
--- a/src/test/scala/com/spark_helper/DateHelperTest.scala
+++ b/src/test/scala/com/spark_helper/DateHelperTest.scala
@@ -2,6 +2,8 @@ package com.spark_helper
 
 import org.scalatest.FunSuite
 
+import com.spark_helper.{DateHelper => DH}
+
 /** Testing facility for date helpers.
   *
   * @author Xavier Guihot
@@ -37,25 +39,20 @@ class DateHelperTest extends FunSuite {
   }
 
   test("Reformat date") {
-    assert(
-      DateHelper.reformatDate("20170327", "yyyyMMdd", "yyMMdd") === "170327")
-    assert(
-      DateHelper.reformatDate("20170327", "yyyyMMdd", "MMddyy") === "032717")
+    assert(DH.reformatDate("20170327", "yyyyMMdd", "yyMMdd") === "170327")
+    assert(DH.reformatDate("20170327", "yyyyMMdd", "MMddyy") === "032717")
   }
 
   test("Next day") {
-    assert(DateHelper.nextDay("20170310") === "20170311")
-    assert(DateHelper.nextDay("170310", "yyMMdd") === "170311")
-    assert(
-      DateHelper.nextDay("20170310_0000", "yyyyMMdd_HHmm") === "20170311_0000")
+    assert(DH.nextDay("20170310") === "20170311")
+    assert(DH.nextDay("170310", "yyMMdd") === "170311")
+    assert(DH.nextDay("20170310_0000", "yyyyMMdd_HHmm") === "20170311_0000")
   }
 
   test("Previous day") {
-    assert(DateHelper.previousDay("20170310") === "20170309")
-    assert(DateHelper.previousDay("170310", "yyMMdd") === "170309")
-    assert(
-      DateHelper
-        .previousDay("20170310_0000", "yyyyMMdd_HHmm") === "20170309_0000")
+    assert(DH.previousDay("20170310") === "20170309")
+    assert(DH.previousDay("170310", "yyMMdd") === "170309")
+    assert(DH.previousDay("20170310_0000", "yyyyMMdd_HHmm") === "20170309_0000")
   }
 
   test("Nbr of days between two dates") {

From 75eadc82a2c6b70bd556ee6156c453e43889c9c5 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Tue, 12 Jun 2018 23:03:02 +0100
Subject: [PATCH 18/25] Pimp Seq[String] and String with a writeToHdfs method

---
 README.md                                     |  8 ++--
 .../scala/com/spark_helper/HdfsHelper.scala   | 39 ++++++++++++++++++-
 .../com/spark_helper/HdfsHelperTest.scala     | 28 +++++++++++--
 .../com/spark_helper/SparkHelperTest.scala    |  4 +-
 4 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index e36dad4..c17d463 100644
--- a/README.md
+++ b/README.md
@@ -18,14 +18,14 @@ names are self-explanatory and readable.
 
 This also provides a monitoring/logger tool.
 
-This is a bunch of 4 modules:
+This is a set of 4 modules:
 
-* [HdfsHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.HdfsHelper$): Wrapper around [apache Hadoop FileSystem API](https://hadoop.apache.org/docs/r2.6.1/api/org/apache/hadoop/fs/FileSystem.html) for file manipulations on hdfs.
-* [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$): Hdfs file manipulations through the Spark API.
+* [HdfsHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.HdfsHelper$): Wrapper around the [apache Hadoop FileSystem API](https://hadoop.apache.org/docs/r2.6.1/api/org/apache/hadoop/fs/FileSystem.html) for file manipulations on hdfs.
+* [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$): Hdfs file manipulations through the Spark API (pimped RDDs and SparkContext).
 * [DateHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.DateHelper$): Wrapper around [joda-time](http://www.joda.org/joda-time/apidocs/) for usual data mining dates manipulations.
 * [Monitor](http://xavierguihot.com/spark_helper/#com.spark_helper.Monitor$): Spark custom monitoring/logger and kpi validator.
 
-Compatible with Spark 2.
+Compatible with Spark 2.x
 
 
 ### HdfsHelper:
diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index a9038bd..82d7f3e 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -6,6 +6,8 @@ import org.apache.hadoop.io.compress.{CompressionCodec, CompressionCodecFactory}
 import org.apache.hadoop.io.compress.{GzipCodec, BZip2Codec}
 import org.apache.hadoop.io.IOUtils
 
+import scala.reflect.ClassTag
+
 import org.joda.time.{DateTime, Days}
 import org.joda.time.format.DateTimeFormat
 
@@ -77,6 +79,41 @@ import com.typesafe.config.{Config, ConfigFactory}
   */
 object HdfsHelper extends Serializable {
 
+  implicit class SeqExtensions[T <: Seq[String]: ClassTag](val seq: T) {
+
+    /** Saves list elements in a file on hdfs.
+      *
+      * Please only consider this way of storing data when the data set is small
+      * enough.
+      *
+      * Overwrites the file if it already exists.
+      *
+      * {{{
+      * Array("some", "relatively small", "text").writeToHdfs("/some/hdfs/file/path.txt")
+      * List("some", "relatively small", "text").writeToHdfs("/some/hdfs/file/path.txt")
+      * }}}
+      *
+      * @param filePath the path of the file in which to write the content of
+      * the List.
+      */
+    def writeToHdfs(filePath: String): Unit =
+      HdfsHelper.writeToHdfsFile(seq, filePath)
+  }
+
+  implicit class StringExtensions(val string: String) {
+
+    /** Saves the String in a file on hdfs.
+      *
+      * Overwrites the file if it already exists.
+      *
+      * {{{ "some\nrelatively small\ntext".writeToHdfsFile("/some/hdfs/file/path.txt") }}}
+      *
+      * @param filePath the path of the file in which to write the String
+      */
+    def writeToHdfs(filePath: String): Unit =
+      HdfsHelper.writeToHdfsFile(string, filePath)
+  }
+
   /** Deletes a file on HDFS.
     *
     * Doesn't throw an exception if the file to delete doesn't exist.
@@ -311,7 +348,7 @@ object HdfsHelper extends Serializable {
     *   List("some", "relatively small", "text"), "/some/hdfs/file/path.txt")
     * }}}
     *
-    * @param content the array of strings to write in the file as one line per
+    * @param content the seq of strings to write in the file as one line per
     * string (this takes care of joining strings with "\n"s).
     * @param filePath the path of the file in which to write the content
     */
diff --git a/src/test/scala/com/spark_helper/HdfsHelperTest.scala b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
index 9babb0b..f96db97 100644
--- a/src/test/scala/com/spark_helper/HdfsHelperTest.scala
+++ b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
@@ -1,5 +1,7 @@
 package com.spark_helper
 
+import com.spark_helper.HdfsHelper._
+
 import org.apache.hadoop.io.compress.GzipCodec
 
 import com.holdenkarau.spark.testing.SharedSparkContext
@@ -127,9 +129,9 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
     val filePath = s"$testFolder/small_file.txt"
 
-    // 1: Stores using a "\n"-joined string:
+    HdfsHelper.deleteFolder(testFolder)
 
-    HdfsHelper.deleteFile(filePath)
+    // 1: Stores using a "\n"-joined string:
 
     val contentToStore = "Hello World\nWhatever"
 
@@ -144,8 +146,6 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
     // 2: Stores using a list of strings to be "\n"-joined:
 
-    HdfsHelper.deleteFile(filePath)
-
     val listToStore = List("Hello World", "Whatever")
     HdfsHelper.writeToHdfsFile(listToStore, filePath)
 
@@ -155,6 +155,26 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     assert(storedContent === listToStore.mkString("\n"))
 
     HdfsHelper.deleteFolder(testFolder)
+
+    // 3: Using the pimped Seq/String:
+
+    listToStore.toSeq.writeToHdfs(filePath)
+    assert(HdfsHelper.fileExists(filePath))
+    storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
+    assert(storedContent === contentToStore)
+    HdfsHelper.deleteFolder(testFolder)
+
+    listToStore.writeToHdfs(filePath)
+    assert(HdfsHelper.fileExists(filePath))
+    storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
+    assert(storedContent === contentToStore)
+    HdfsHelper.deleteFolder(testFolder)
+
+    contentToStore.writeToHdfs(filePath)
+    assert(HdfsHelper.fileExists(filePath))
+    storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
+    assert(storedContent === contentToStore)
+    HdfsHelper.deleteFolder(testFolder)
   }
 
   test("List file names in Hdfs folder") {
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 1ccfba4..21d01e0 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -1,8 +1,6 @@
 package com.spark_helper
 
-import com.spark_helper.SparkHelper.{RDDExtensions, StringRDDExtensions}
-import com.spark_helper.SparkHelper.{SeqRDDExtensions, OptionRDDExtensions}
-import com.spark_helper.SparkHelper.{SparkContextExtensions, PairRDDExtensions}
+import com.spark_helper.SparkHelper._
 
 import org.apache.hadoop.io.compress.GzipCodec
 

From 889d5e53caab48de427cb1a5ea2fe64f06d8592b Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Wed, 13 Jun 2018 22:27:12 +0100
Subject: [PATCH 19/25] Make a singleton out of HdfsHelper by allowing setting
 a specific Configuration or FileSystem

---
 .../scala/com/spark_helper/HdfsHelper.scala   | 141 ++++++++----------
 1 file changed, 66 insertions(+), 75 deletions(-)

diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index 82d7f3e..694131e 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -79,6 +79,33 @@ import com.typesafe.config.{Config, ConfigFactory}
   */
 object HdfsHelper extends Serializable {
 
+  private var conf = new Configuration()
+  private var hdfs = FileSystem.get(conf)
+
+  /** Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Configuration</code>
+    * used by the underlying <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+    * in case it requires some specificities.
+    *
+    * If this setter is not used, the default Configuration is set with
+    * <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">new Configuration()</code>.
+    *
+    * @param conf the specific Configuration to use
+    */
+  def setConf(configuration: Configuration): Unit = {
+    conf = configuration
+    hdfs = FileSystem.get(configuration)
+  }
+
+  /** Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+    * in case it requires some specificities.
+    *
+    * If this setter is not used, the default FileSystem is set with
+    * <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem.get(new Configuration())</code>.
+    *
+    * @param fileSystem the specific FileSystem to use
+    */
+  def setFileSystem(fileSystem: FileSystem): Unit = hdfs = fileSystem
+
   implicit class SeqExtensions[T <: Seq[String]: ClassTag](val seq: T) {
 
     /** Saves list elements in a file on hdfs.
@@ -122,17 +149,15 @@ object HdfsHelper extends Serializable {
     */
   def deleteFile(hdfsPath: String): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val fileToDelete = new Path(hdfsPath)
 
-    if (fileSystem.exists(fileToDelete)) {
+    if (hdfs.exists(fileToDelete)) {
 
       require(
-        fileSystem.isFile(fileToDelete),
+        hdfs.isFile(fileToDelete),
         "to delete a folder, prefer using the deleteFolder() method.")
 
-      fileSystem.delete(fileToDelete, true)
+      hdfs.delete(fileToDelete, true)
     }
   }
 
@@ -144,17 +169,15 @@ object HdfsHelper extends Serializable {
     */
   def deleteFolder(hdfsPath: String): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val folderToDelete = new Path(hdfsPath)
 
-    if (fileSystem.exists(folderToDelete)) {
+    if (hdfs.exists(folderToDelete)) {
 
       require(
-        !fileSystem.isFile(folderToDelete),
+        !hdfs.isFile(folderToDelete),
         "to delete a file, prefer using the deleteFile() method.")
 
-      fileSystem.delete(folderToDelete, true)
+      hdfs.delete(folderToDelete, true)
     }
   }
 
@@ -164,8 +187,7 @@ object HdfsHelper extends Serializable {
     *
     * @param hdfsPath the path of the folder to create
     */
-  def createFolder(hdfsPath: String): Unit =
-    FileSystem.get(new Configuration()).mkdirs(new Path(hdfsPath))
+  def createFolder(hdfsPath: String): Unit = hdfs.mkdirs(new Path(hdfsPath))
 
   /** Checks if the file exists.
     *
@@ -174,16 +196,14 @@ object HdfsHelper extends Serializable {
     */
   def fileExists(hdfsPath: String): Boolean = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val fileToCheck = new Path(hdfsPath)
 
-    if (fileSystem.exists(fileToCheck))
+    if (hdfs.exists(fileToCheck))
       require(
-        fileSystem.isFile(fileToCheck),
+        hdfs.isFile(fileToCheck),
         "to check if a folder exists, prefer using the folderExists() method.")
 
-    fileSystem.exists(fileToCheck)
+    hdfs.exists(fileToCheck)
   }
 
   /** Checks if the folder exists.
@@ -193,16 +213,14 @@ object HdfsHelper extends Serializable {
     */
   def folderExists(hdfsPath: String): Boolean = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val folderToCheck = new Path(hdfsPath)
 
-    if (fileSystem.exists(folderToCheck))
+    if (hdfs.exists(folderToCheck))
       require(
-        !fileSystem.isFile(folderToCheck),
+        !hdfs.isFile(folderToCheck),
         "to check if a file exists, prefer using the fileExists() method.")
 
-    fileSystem.exists(folderToCheck)
+    hdfs.exists(folderToCheck)
   }
 
   /** Moves/renames a file.
@@ -221,21 +239,19 @@ object HdfsHelper extends Serializable {
       overwrite: Boolean = false
   ): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val fileToRename = new Path(oldPath)
     val renamedFile = new Path(newPath)
 
-    if (fileSystem.exists(fileToRename))
+    if (hdfs.exists(fileToRename))
       require(
-        fileSystem.isFile(fileToRename),
+        hdfs.isFile(fileToRename),
         "to move a folder, prefer using the moveFolder() method.")
 
     if (overwrite)
-      fileSystem.delete(renamedFile, true)
+      hdfs.delete(renamedFile, true)
     else
       require(
-        !fileSystem.exists(renamedFile),
+        !hdfs.exists(renamedFile),
         "overwrite option set to false, but a file already exists at target " +
           "location " + newPath)
 
@@ -244,7 +260,7 @@ object HdfsHelper extends Serializable {
     val targetContainerFolder = newPath.split("/").init.mkString("/")
     createFolder(targetContainerFolder)
 
-    fileSystem.rename(fileToRename, renamedFile)
+    hdfs.rename(fileToRename, renamedFile)
   }
 
   /** Moves/renames a folder.
@@ -263,21 +279,19 @@ object HdfsHelper extends Serializable {
       overwrite: Boolean = false
   ): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val folderToRename = new Path(oldPath)
     val renamedFolder = new Path(newPath)
 
-    if (fileSystem.exists(folderToRename))
+    if (hdfs.exists(folderToRename))
       require(
-        !fileSystem.isFile(folderToRename),
+        !hdfs.isFile(folderToRename),
         "to move a file, prefer using the moveFile() method.")
 
     if (overwrite)
-      fileSystem.delete(renamedFolder, true)
+      hdfs.delete(renamedFolder, true)
     else
       require(
-        !fileSystem.exists(renamedFolder),
+        !hdfs.exists(renamedFolder),
         "overwrite option set to false, but a folder already exists at target " +
           "location " + newPath)
 
@@ -286,7 +300,7 @@ object HdfsHelper extends Serializable {
     val targetContainerFolder = newPath.split("/").init.mkString("/")
     createFolder(targetContainerFolder)
 
-    fileSystem.rename(folderToRename, new Path(newPath))
+    hdfs.rename(folderToRename, new Path(newPath))
   }
 
   /** Creates an empty file on hdfs.
@@ -309,7 +323,7 @@ object HdfsHelper extends Serializable {
     * @param filePath the path of the empty file to create
     */
   def createEmptyHdfsFile(filePath: String): Unit =
-    FileSystem.get(new Configuration()).create(new Path(filePath)).close()
+    hdfs.create(new Path(filePath)).close()
 
   /** Saves text in a file when content is too small to really require an RDD.
     *
@@ -326,10 +340,7 @@ object HdfsHelper extends Serializable {
     * @param filePath the path of the file in which to write the content
     */
   def writeToHdfsFile(content: String, filePath: String): Unit = {
-
-    val outputFile =
-      FileSystem.get(new Configuration()).create(new Path(filePath))
-
+    val outputFile = hdfs.create(new Path(filePath))
     outputFile.write(content.getBytes("UTF-8"))
     outputFile.close()
   }
@@ -374,8 +385,7 @@ object HdfsHelper extends Serializable {
       onlyName: Boolean = true
   ): List[String] = {
 
-    FileSystem
-      .get(new Configuration())
+    hdfs
       .listStatus(new Path(hdfsPath))
       .flatMap { status =>
         // If it's a file:
@@ -408,8 +418,7 @@ object HdfsHelper extends Serializable {
     * @return the list of folder names in the specified folder
     */
   def listFolderNamesInFolder(hdfsPath: String): List[String] =
-    FileSystem
-      .get(new Configuration())
+    hdfs
       .listStatus(new Path(hdfsPath))
       .filter(!_.isFile)
       .map(_.getPath.getName)
@@ -423,11 +432,7 @@ object HdfsHelper extends Serializable {
     * @return the joda DateTime of the last modification of the given file
     */
   def fileModificationDateTime(hdfsPath: String): DateTime =
-    new DateTime(
-      FileSystem
-        .get(new Configuration())
-        .getFileStatus(new Path(hdfsPath))
-        .getModificationTime())
+    new DateTime(hdfs.getFileStatus(new Path(hdfsPath)).getModificationTime())
 
   /** Returns the stringified date of the last modification of the given file.
     *
@@ -599,9 +604,7 @@ object HdfsHelper extends Serializable {
     */
   def validateHdfsXmlWithXsd(hdfsXmlPath: String, xsdFile: URL): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
-    val xmlFile = new StreamSource(fileSystem.open(new Path(hdfsXmlPath)))
+    val xmlFile = new StreamSource(hdfs.open(new Path(hdfsXmlPath)))
 
     val schemaFactory =
       SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI)
@@ -644,10 +647,7 @@ object HdfsHelper extends Serializable {
     * @return the com.typesafe.config.Config object which contains usable data
     */
   def loadTypesafeConfigFromHdfs(hdfsConfigPath: String): Config = {
-
-    val reader = new InputStreamReader(
-      FileSystem.get(new Configuration()).open(new Path(hdfsConfigPath)))
-
+    val reader = new InputStreamReader(hdfs.open(new Path(hdfsConfigPath)))
     try { ConfigFactory.parseReader(reader) } finally { reader.close() }
   }
 
@@ -660,10 +660,7 @@ object HdfsHelper extends Serializable {
     * @return the scala.xml.Elem object
     */
   def loadXmlFileFromHdfs(hdfsXmlPath: String): Elem = {
-
-    val reader = new InputStreamReader(
-      FileSystem.get(new Configuration()).open(new Path(hdfsXmlPath)))
-
+    val reader = new InputStreamReader(hdfs.open(new Path(hdfsXmlPath)))
     try { XML.load(reader) } finally { reader.close() }
   }
 
@@ -692,8 +689,6 @@ object HdfsHelper extends Serializable {
       deleteInputFile: Boolean = true
   ): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val ClassOfGzip = classOf[GzipCodec]
     val ClassOfBZip2 = classOf[BZip2Codec]
 
@@ -702,12 +697,11 @@ object HdfsHelper extends Serializable {
       case ClassOfBZip2 => s"$inputPath.bz2"
     }
 
-    val inputStream = fileSystem.open(new Path(inputPath))
-    val outputStream = fileSystem.create(new Path(outputPath))
+    val inputStream = hdfs.open(new Path(inputPath))
+    val outputStream = hdfs.create(new Path(outputPath))
 
     // The compression code:
-    val codec = new CompressionCodecFactory(new Configuration())
-      .getCodec(new Path(outputPath))
+    val codec = new CompressionCodecFactory(conf).getCodec(new Path(outputPath))
     // We include the compression codec to the output stream:
     val compressedOutputStream = codec.createOutputStream(outputStream)
 
@@ -715,7 +709,7 @@ object HdfsHelper extends Serializable {
       IOUtils.copyBytes(
         inputStream,
         compressedOutputStream,
-        new Configuration(),
+        conf,
         false
       )
     } finally {
@@ -746,8 +740,7 @@ object HdfsHelper extends Serializable {
       purgeAge >= 0,
       "the purgeAge provided \"" + purgeAge.toString + "\" must be superior to 0.")
 
-    FileSystem
-      .get(new Configuration())
+    hdfs
       .listStatus(new Path(folderPath))
       .filter(path => {
 
@@ -781,22 +774,20 @@ object HdfsHelper extends Serializable {
       workingFolderPath: String
   ): Unit = {
 
-    val fileSystem = FileSystem.get(new Configuration())
-
     val tmpOutputPath = workingFolderPath match {
       case "" => s"$filePath.tmp"
       case _  => s"$workingFolderPath/xml.tmp"
     }
     deleteFile(tmpOutputPath)
 
-    val inputFile = fileSystem.open(new Path(filePath))
-    val tmpOutputFile = fileSystem.create(new Path(tmpOutputPath))
+    val inputFile = hdfs.open(new Path(filePath))
+    val tmpOutputFile = hdfs.create(new Path(tmpOutputPath))
 
     // If there is an header, we add it to the file:
     header.foreach(h => tmpOutputFile.write((h + "\n").getBytes("UTF-8")))
 
     try {
-      IOUtils.copyBytes(inputFile, tmpOutputFile, new Configuration(), false)
+      IOUtils.copyBytes(inputFile, tmpOutputFile, conf, false)
     } finally {
       inputFile.close()
     }

From 5351664daa9e164e57e0c8ed69b8dd4d9e2db4fc Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 14 Jun 2018 21:01:32 +0100
Subject: [PATCH 20/25] Improve README

---
 README.md                                     | 66 +++++++++++++------
 .../scala/com/spark_helper/HdfsHelper.scala   |  1 +
 .../scala/com/spark_helper/SparkHelper.scala  | 21 +++---
 3 files changed, 57 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index c17d463..5d6fed8 100644
--- a/README.md
+++ b/README.md
@@ -36,21 +36,21 @@ The full list of methods is available at
 Contains basic file-related methods mostly based on hdfs apache Hadoop
 FileSystem API [org.apache.hadoop.fs.FileSystem](https://hadoop.apache.org/docs/r2.6.1/api/org/apache/hadoop/fs/FileSystem.html).
 
-For instance, one don't want to remove a file from hdfs using 3 lines of code
-and thus could instead just use `HdfsHelper.deleteFile("my/hdfs/file/path.csv")`.
-
 A non-exhaustive list of exemples:
 
 ```scala
 import com.spark_helper.HdfsHelper
 
 // A bunch of methods wrapping the FileSystem API, such as:
-HdfsHelper.fileExists("my/hdfs/file/path.txt")
+HdfsHelper.fileExists("my/hdfs/file/path.txt") // HdfsHelper.folderExists("my/hdfs/folder")
 assert(HdfsHelper.listFileNamesInFolder("my/folder/path") == List("file_name_1.txt", "file_name_2.csv"))
 assert(HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") == "20170306")
 assert(HdfsHelper.nbrOfDaysSinceFileWasLastModified("my/hdfs/file/path.txt") == 3)
-HdfsHelper.deleteFile("my/hdfs/file/path.csv")
-HdfsHelper.moveFolder("my/hdfs/folder")
+HdfsHelper.deleteFile("my/hdfs/file/path.csv") // HdfsHelper.deleteFolder("my/hdfs/folder")
+HdfsHelper.moveFolder("my/hdfs/folder") // HdfsHelper.moveFile("my/hdfs/file.txt")
+HdfsHelper.createEmptyHdfsFile("/some/hdfs/file/path.token") // HdfsHelper.createFolder("my/hdfs/folder")
+
+// File content helpers:
 HdfsHelper.compressFile("hdfs/path/to/uncompressed_file.txt", classOf[GzipCodec])
 HdfsHelper.appendHeader("my/hdfs/file/path.csv", "colum0,column1")
 
@@ -63,41 +63,67 @@ HdfsHelper.loadTypesafeConfigFromHdfs("my/hdfs/file/path.conf"): Config
 
 // In order to write small amount of data in a file on hdfs without the whole spark stack:
 HdfsHelper.writeToHdfsFile(Array("some", "relatively small", "text"), "/some/hdfs/file/path.txt")
+// or:
+import com.spark_helper.HdfsHelper._
+Array("some", "relatively small", "text").writeToHdfs("/some/hdfs/file/path.txt")
+"hello world".writeToHdfs("/some/hdfs/file/path.txt")
 
 // Deletes all files/folders in "hdfs/path/to/folder" for which the timestamp is older than 10 days:
 HdfsHelper.purgeFolder("hdfs/path/to/folder", 10)
 ```
 
+In case a specific configuration is needed to access the file system, these
+setters are available:
+
+```scala
+// To use a specific conf FileSystem.get(whateverConf) instead of FileSystem.get(new Configuration()):
+HdfsHelper.setConf(whateverConf)
+// Or directly the FileSystem:
+HdfsHelper.setFileSystem(whateverFileSystem)
+```
+
 ### SparkHelper:
 
 The full list of methods is available at
 [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$).
 
-Contains basic file/RRD-related methods based on the Spark APIs.
+Contains basic RRD-related methods.
 
 A non-exhaustive list of exemples:
 
 ```scala
-import com.spark_helper.SparkHelper
+import com.spark_helper.SparkHelper._
 
-// Same as SparkContext.saveAsTextFile, but the result is a single file:
-SparkHelper.saveAsSingleTextFile(myOutputRDD, "/my/output/file/path.txt")
+// Same as rdd.saveAsTextFile("path"), but the result is a single file (while
+// keeping the processing parallelized):
+rdd.saveAsSingleTextFile("/my/output/file/path.txt")
+rdd.saveAsSingleTextFile("/my/output/file/path.txt", classOf[BZip2Codec])
 
-// Same as SparkContext.textFile, but instead of reading one record per line,
-// it reads records spread over several lines. This way, xml, json, yml or
-// any multi-line record file format can be used with Spark:
-SparkHelper.textFileWithDelimiter("/my/input/folder/path", sparkContext, "---\n")
+// Same as sc.textFile("path"), but instead of reading one record per line (by
+// splitting the input with \n), it splits the file in records based on a custom
+// delimiter. This way, xml, json, yml or any multi-line record file format can
+// be used with Spark:
+sc.textFile("/my/input/folder/path", "---\n")
+
+// Equivalent to rdd.flatMap(identity) for RDDs of Seqs or Options:
+rdd.flatten
 
 // Equivalent to sparkContext.textFile(), but for each line is tupled with its
 // file path:
 SparkHelper.textFileWithFileName("folder", sparkContext)
 // which produces:
-RDD(
-    ("file:/path/on/machine/folder/file_1.txt", "record1fromfile1"),
-    ("file:/path/on/machine/folder/file_1.txt", "record2fromfile1"),
-    ("file:/path/on/machine/folder/file_2.txt", "record1fromfile2"),
-    ...
-)
+// RDD(("folder/file_1.txt", "record1fromfile1"), ("folder/file_1.txt", "record2fromfile1"),
+//    ("folder/file_2.txt", "record1fromfile2"), ...)
+
+// In the given folder, this generates one file per key in the given key/value
+// RDD. Within each file (named from the key) are all values for this key:
+rdd.saveAsTextFileByKey("/my/output/folder/path")
+
+// Concept mapper (the following exemple transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
+rdd.partialMap { case a if a % 2 == 0 => 2 * a }
+
+// For when input files contain commas and textFile can't handle it:
+sc.textFile(Seq("path/hello,world.txt", "path/hello_world.txt"))
 ```
 
 ### DateHelper:
diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index 694131e..920c905 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -74,6 +74,7 @@ import com.typesafe.config.{Config, ConfigFactory}
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/HdfsHelper.scala">HdfsHelper</a>
   *
+  * @todo Create a touch method
   * @author Xavier Guihot
   * @since 2017-02
   */
diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 075c22f..336ff4a 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -512,17 +512,16 @@ object SparkHelper extends Serializable {
             iterator.map(tpl => (file.getPath.toString, tpl._2.toString))
         }
 
-      /* An other way of doing would be:
-     *
-     * import org.apache.spark.sql.functions.input_file_name
-     * import spark.implicits._
-     *
-     * spark.read
-     *   .text(testFolder)
-     *   .select(input_file_name, $"value")
-     *   .as[(String, String)]
-     *   .rdd
-     */
+      // An other way of doing would be:
+      //
+      // import org.apache.spark.sql.functions.input_file_name
+      // import spark.implicits._
+      //
+      // spark.read
+      //   .text(testFolder)
+      //   .select(input_file_name, $"value")
+      //   .as[(String, String)]
+      //   .rdd
     }
 
     /** A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>

From b812cc1e70308cd931697ea2a0119360da768222 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 14 Jun 2018 23:06:22 +0100
Subject: [PATCH 21/25] Pimp String/Int with DateHelper functions - refactor
 DateHelper/README

---
 README.md                                     |  39 ++-
 .../scala/com/spark_helper/DateHelper.scala   | 304 ++++++++++++++----
 .../com/spark_helper/DateHelperTest.scala     |  12 +
 3 files changed, 290 insertions(+), 65 deletions(-)

diff --git a/README.md b/README.md
index 5d6fed8..9ec5927 100644
--- a/README.md
+++ b/README.md
@@ -132,21 +132,42 @@ The full list of methods is available at
 [DateHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.DateHelper$).
 
 Wrapper around [joda-time](http://www.joda.org/joda-time/apidocs/) for
-data-mining classic dates manipulations.
+data-mining classic dates manipulations and job scheduling.
 
 A non-exhaustive list of exemples:
 
 ```scala
 import com.spark_helper.DateHelper
 
-assert(DateHelper.daysBetween("20161230", "20170101") == List("20161230", "20161231", "20170101"))
-assert(DateHelper.today() == "20170310") // If today's "20170310"
-assert(DateHelper.yesterday() == "20170309") // If today's "20170310"
-assert(DateHelper.reformatDate("20170327", "yyyyMMdd", "yyMMdd") == "170327")
-assert(DateHelper.now("HH:mm") == "10:24")
-assert(DateHelper.currentTimestamp() == "1493105229736")
-assert(DateHelper.nDaysBefore(3) == "20170307") // If today's "20170310"
-assert(DateHelper.nDaysAfterDate(3, "20170307") == "20170310")
+DateHelper.daysBetween("20161230", "20170101") // List("20161230", "20161231", "20170101")
+DateHelper.today // "20170310"
+DateHelper.yesterday // "20170309"
+DateHelper.reformatDate("20170327", "yyyyMMdd", "yyMMdd") // "170327"
+DateHelper.now("HH:mm") // "10:24"
+DateHelper.currentTimestamp // "1493105229736"
+DateHelper.nDaysBefore(3) // "20170307"
+DateHelper.nDaysAfterDate(3, "20170307") // "20170310"
+DateHelper.nextDay("20170310") // "20170311"
+DateHelper.nbrOfDaysSince("20170302") // 8
+DateHelper.nbrOfDaysBetween("20170327", "20170401") // 5
+DateHelper.dayOfWeek("20160614") // 2
+
+import com.spark_helper.DateHelper._
+
+2.daysAgo // "20170308"
+3.daysBefore("20170310") // "20170307"
+5.daysAfter // "20170315"
+4.daysAfter("20170310") // "20170314"
+"20170302".isCompliantWith("yyyyMMdd")
+"20170310".nextDay // "20170311"
+"20170310".previousDay // "20170309"
+```
+
+The default format (when no format is specified) is "yyyyMMdd" (20170327). It
+can be modified globally with:
+
+```scala
+DateHelper.setFormat("ddMMMyy")
 ```
 
 ### Monitor:
diff --git a/src/main/scala/com/spark_helper/DateHelper.scala b/src/main/scala/com/spark_helper/DateHelper.scala
index c2af405..9d11627 100644
--- a/src/main/scala/com/spark_helper/DateHelper.scala
+++ b/src/main/scala/com/spark_helper/DateHelper.scala
@@ -28,11 +28,126 @@ import scala.util.Try
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/DateHelper.scala">DateHelper</a>
   *
+  * @todo Is it possible to ddo something like ("20170325" to "20170327")?
   * @author Xavier Guihot
   * @since 2017-02
   */
 object DateHelper extends Serializable {
 
+  private var defaultFormat = "yyyyMMdd"
+
+  /** Sets the default date format used by these functions when no date format
+    * is specified.
+    *
+    * {{{
+    * // By default, yyyyMMdd is used:
+    * assert(3.daysBefore == "20170307")
+    * // But this can be modified globally:
+    * DateHelper.setFormat("ddMMMyy")
+    * assert(3.daysBefore == "07Mar17")
+    * }}}
+    *
+    * @param format the new default format
+    */
+  def setFormat(format: String): Unit = defaultFormat = format
+
+  implicit class IntExtensions(val int: Int) {
+
+    /** Returns which date it was x days before today under the default format.
+      *
+      * If we're "20170125" and we request for 3 days before, we'll return
+      * "20170122".
+      *
+      * {{{
+      * // If today's "20170310":
+      * assert(3.daysAgo == "20170307")
+      * }}}
+      *
+      * @return today's date minus the given nbr of days
+      */
+    def daysAgo(): String = DateHelper.nDaysBefore(int)
+
+    /** Returns which date it was x days before the given date.
+      *
+      * If the given date is "20170125" and we request the date it was 3 days
+      * before, this will return "20170122".
+      *
+      * {{{ assert(3.daysBefore("20170310") == "20170307") }}}
+      *
+      * @param date the date under the default format for which we want the date
+      * for nbrOfDaysBefore days before.
+      * @return the date it was nbrOfDaysBefore before date under the default
+      * format.
+      */
+    def daysBefore(date: String): String = DateHelper.nDaysBeforeDate(int, date)
+
+    /** Returns which date it will be x days after today under the default format.
+      *
+      * If we're "20170125" and we request for 3 days after, we'll return
+      * "20170127".
+      *
+      * {{{
+      * // If today's "20170310":
+      * assert(3.daysAfter == "20170313")
+      * }}}
+      *
+      * @return today's date plus the given nbr of days
+      */
+    def daysAfter(): String = DateHelper.nDaysAfter(int)
+
+    /** Returns which date it will be x days after the given date under the
+      * default format.
+      *
+      * If the given date is "20170122" and we request the date it will be 3
+      * days after, we'll return "20170125".
+      *
+      * {{{ assert(5.daysAfter("20170305") == "20170310") }}}
+      *
+      * @param date the date under the default format for which we want the date
+      * for nbrOfDaysAfter days after.
+      * @return the date it was nbrOfDaysAfter after date under the default
+      * format.
+      */
+    def daysAfter(date: String): String = DateHelper.nDaysAfterDate(int, date)
+  }
+
+  implicit class StringExtensions(val string: String) {
+
+    /** Validates the stringified date is compliant with the provided format.
+      *
+      * {{{
+      * assert("20170302".isCompliantWith("yyyyMMdd"))
+      * assert(!"20170333".isCompliantWith("yyyyMMdd"))
+      * assert("20170228".isCompliantWith("yyyyMMdd"))
+      * assert(!"20170229".isCompliantWith("yyyyMMdd"))
+      * assert(!"170228".isCompliantWith("yyyyMMdd"))
+      * assert(!"".isCompliantWith("yyyyMMdd"))
+      * assert(!"a".isCompliantWith("yyyyMMdd"))
+      * assert(!"24JAN17".isCompliantWith("yyyyMMdd"))
+      * }}}
+      *
+      * @return if the provided date is under the provided format
+      */
+    def isCompliantWith(format: String): Boolean =
+      DateHelper.isDateCompliantWithFormat(string, format)
+
+    /** Returns the date one day after the given date.
+      *
+      * {{{ assert("20170310".nextDay == "20170311") }}}
+      *
+      * @return the date of the day after the given date
+      */
+    def nextDay(): String = DateHelper.nextDay(string)
+
+    /** Returns the date one day before the given date.
+      *
+      * {{{ assert("20170310".previousDay == "20170309") }}}
+      *
+      * @return the date of the day before the given date
+      */
+    def previousDay(): String = DateHelper.previousDay(string)
+  }
+
   /** Finds the list of dates between the two given dates.
     *
     * {{{
@@ -84,47 +199,78 @@ object DateHelper extends Serializable {
   }
 
   /** Returns which date it was x days before today under the requested format.
-    *
-    * If we're "20170125" and we request for 3 days before, we'll return
-    * "20170122".
     *
     * {{{
     * // If today's "20170310":
-    * assert(DateHelper.nDaysBefore(3) == "20170307")
     * assert(DateHelper.nDaysBefore(5, "yyMMdd") == "170305")
     * }}}
     *
     * @param nbrOfDaysBefore the nbr of days before today
-    * @param format (default = "yyyyMMdd") the format for the returned date
+    * @param format the format for the returned date
     * @return today's date minus the nbrOfDaysBefore under the requested format
     */
-  def nDaysBefore(nbrOfDaysBefore: Int, format: String = "yyyyMMdd"): String =
+  def nDaysBefore(nbrOfDaysBefore: Int, format: String): String =
     DateTimeFormat
       .forPattern(format)
       .print(new DateTime().minusDays(nbrOfDaysBefore))
 
-  /** Returns which date it was x days before the given date.
+  /** Returns which date it was x days before today.
     *
-    * If the given date is "20170125" and we request the date it was 3 days
-    * before, we'll return "20170122".
+    * {{{
+    * // If today's "20170310":
+    * assert(DateHelper.nDaysBefore(5) == "20170305")
+    * }}}
+    *
+    * @param nbrOfDaysBefore the nbr of days before today
+    * @return today's date minus the nbrOfDaysBefore under the default format
+    */
+  def nDaysBefore(nbrOfDaysBefore: Int): String =
+    nDaysBefore(nbrOfDaysBefore, defaultFormat)
+
+  /** Returns which date it will be x days after today under the requested format.
+    *
+    * {{{
+    * // If today's "20170310":
+    * assert(DateHelper.nDaysAfter(5, "yyMMdd") == "170315")
+    * }}}
+    *
+    * @param nbrOfDaysAfter the nbr of days after today
+    * @param format the format for the returned date
+    * @return today's date plus the nbrOfDaysAfter under the requested format
+    */
+  def nDaysAfter(nbrOfDaysAfter: Int, format: String): String =
+    nDaysBefore(-nbrOfDaysAfter, format)
+
+  /** Returns which date it will be x days after today under the default format.
     *
     * {{{
-    * assert(DateHelper.nDaysBeforeDate(3, "20170310") == "20170307")
-    * assert(DateHelper.nDaysBeforeDate(5, "170310", "yyMMdd") == "170305")
+    * // If today's "20170310":
+    * assert(DateHelper.nDaysAfter(5) == "20170315")
     * }}}
     *
+    * @param nbrOfDaysAfter the nbr of days after today
+    * @return today's date plus the nbrOfDaysAfter under the default format
+    */
+  def nDaysAfter(nbrOfDaysAfter: Int): String = nDaysBefore(-nbrOfDaysAfter)
+
+  /** Returns which date it was x days before the given date.
+    *
+    * If the given date is "20170125" and we request the date it was 3 days
+    * before, this will return "20170122".
+    *
+    * {{{ assert(DateHelper.nDaysBeforeDate(5, "170310", "yyMMdd") == "170305") }}}
+    *
     * @param nbrOfDaysBefore the nbr of days before the given date
     * @param date the date under the provided format for which we want the date
     * for nbrOfDaysBefore days before.
-    * @param format (default = "yyyyMMdd") the format for the provided and
-    * returned dates.
+    * @param format the format for the provided and returned dates.
     * @return the date it was nbrOfDaysBefore before date under the requested
     * format.
     */
   def nDaysBeforeDate(
       nbrOfDaysBefore: Int,
       date: String,
-      format: String = "yyyyMMdd"
+      format: String
   ): String = {
 
     val currentDate = DateTimeFormat.forPattern(format).parseDateTime(date)
@@ -134,36 +280,59 @@ object DateHelper extends Serializable {
       .print(currentDate.minusDays(nbrOfDaysBefore))
   }
 
+  /** Returns which date it was x days before the given date.
+    *
+    * If the given date is "20170125" and we request the date it was 3 days
+    * before, this will return "20170122".
+    *
+    * {{{ assert(DateHelper.nDaysBeforeDate(5, "20170310") == "20170305") }}}
+    *
+    * @param nbrOfDaysBefore the nbr of days before the given date
+    * @param date the date under the default format for which we want the date
+    * for nbrOfDaysBefore days before.
+    * @return the date it was nbrOfDaysBefore before date under the default
+    * format.
+    */
+  def nDaysBeforeDate(nbrOfDaysBefore: Int, date: String): String =
+    nDaysBeforeDate(nbrOfDaysBefore, date, defaultFormat)
+
   /** Returns which date it will be x days after the given date.
     *
     * If the given date is "20170122" and we request the date it will be 3 days
     * after, we'll return "20170125".
     *
-    * {{{
-    * assert(DateHelper.nDaysAfterDate(3, "20170307") == "20170310")
-    * assert(DateHelper.nDaysAfterDate(5, "170305", "yyMMdd") == "170310")
-    * }}}
+    * {{{ assert(DateHelper.nDaysAfterDate(5, "170305", "yyMMdd") == "170310") }}}
     *
     * @param nbrOfDaysAfter the nbr of days after the given date
     * @param date the date under the provided format for which we want the date
     * for nbrOfDaysAfter days after.
-    * @param format (default = "yyyyMMdd") the format for the provided and
-    * returned dates.
+    * @param format the format for the provided and returned dates.
     * @return the date it was nbrOfDaysAfter after date under the requested
     * format.
     */
   def nDaysAfterDate(
       nbrOfDaysAfter: Int,
       date: String,
-      format: String = "yyyyMMdd"
-  ): String = {
-
-    val currentDate = DateTimeFormat.forPattern(format).parseDateTime(date)
+      format: String
+  ): String =
+    nDaysBeforeDate(-nbrOfDaysAfter, date, format)
 
-    DateTimeFormat
-      .forPattern(format)
-      .print(currentDate.plusDays(nbrOfDaysAfter))
-  }
+  /** Returns which date it will be x days after the given date under the
+    * default format.
+    *
+    * If the given date is "20170122" and we request the date it will be 3 days
+    * after, we'll return "20170125".
+    *
+    * {{{ assert(DateHelper.nDaysAfterDate(5, "20170305") == "20170310") }}}
+    *
+    * @param nbrOfDaysAfter the nbr of days after the given date
+    * @param date the date under the default format for which we want the date
+    * for nbrOfDaysAfter days after.
+    * @return the date it was nbrOfDaysAfter after date under the default
+    * format.
+    */
+  def nDaysAfterDate(nbrOfDaysAfter: Int, date: String): String =
+    nDaysAfterDate(nbrOfDaysAfter, date, defaultFormat)
 
   /** Returns today's date/time under the requested format.
     *
@@ -187,42 +356,70 @@ object DateHelper extends Serializable {
     *
     * {{{
     * // If today's "20170310":
-    * assert(DateHelper.today() == "20170310")
     * assert(DateHelper.today("yyMMdd") == "170310")
     * }}}
     *
-    * @param format (default = "yyyyMMdd") the format for the current date
+    * @param format the format for the current date
     * @return today's date under the requested format
     */
-  def today(format: String = "yyyyMMdd"): String = nDaysBefore(0, format)
+  def today(format: String): String = nDaysBefore(0, format)
+
+  /** Returns today's date/time under the default format.
+    *
+    * {{{
+    * // If today's "20170310":
+    * assert(DateHelper.today() == "20170310")
+    * }}}
+    *
+    * @return today's date under the default format
+    */
+  def today(): String = nDaysBefore(0, defaultFormat)
 
   /** Returns yesterday's date/time under the requested format.
     *
     * {{{
     * // If today's "20170310":
-    * assert(DateHelper.yesterday() == "20170309")
     * assert(DateHelper.yesterday("yyMMdd") == "170309")
     * }}}
     *
-    * @param format (default = "yyyyMMdd") the format in which to output the
-    * date of yesterday.
+    * @param format the format in which to output the date of yesterday
     * @return yesterday's date under the requested format
     */
-  def yesterday(format: String = "yyyyMMdd"): String = nDaysBefore(1, format)
+  def yesterday(format: String): String = nDaysBefore(1, format)
+
+  /** Returns yesterday's date/time under the default format.
+    *
+    * {{{
+    * // If today's "20170310":
+    * assert(DateHelper.yesterday() == "20170309")
+    * }}}
+    *
+    * @return yesterday's date under the default format
+    */
+  def yesterday(): String = nDaysBefore(1, defaultFormat)
 
   /** Returns which date it was 2 days before today under the requested format.
     *
     * {{{
     * // If today's "20170310":
-    * assert(DateHelper.twoDaysAgo() == "20170308")
     * assert(DateHelper.twoDaysAgo("yyMMdd") == "170308")
     * }}}
     *
-    * @param format (default = "yyyyMMdd") the format in which to output the
-    * date of two days ago.
+    * @param format the format in which to output the date of two days ago
     * @return the date of two days ago under the requested format
     */
-  def twoDaysAgo(format: String = "yyyyMMdd"): String = nDaysBefore(2, format)
+  def twoDaysAgo(format: String): String = nDaysBefore(2, format)
+
+  /** Returns which date it was 2 days before today under the default format.
+    *
+    * {{{
+    * // If today's "20170310":
+    * assert(DateHelper.twoDaysAgo() == "20170308")
+    * }}}
+    *
+    * @return the date of two days ago under the default format
+    */
+  def twoDaysAgo(): String = nDaysBefore(2, defaultFormat)
 
   /** Reformats a date from one format to another.
     *
@@ -265,17 +462,15 @@ object DateHelper extends Serializable {
   /** Returns for a date the date one day latter.
     *
     * {{{
-    * // If the given date is "20170310":
     * assert(DateHelper.nextDay("20170310") == "20170311")
     * assert(DateHelper.nextDay("170310", "yyMMdd") == "170311")
     * }}}
     *
     * @param date the date for which to find the date of the day after
-    * @param format (default = "yyyyMMdd") the format of the provided and the
-    * returned dates.
+    * @param format the format of the provided and the returned dates
     * @return the date of the day after the given date
     */
-  def nextDay(date: String, format: String = "yyyyMMdd"): String = {
+  def nextDay(date: String, format: String = defaultFormat): String = {
     val currentDate = DateTimeFormat.forPattern(format).parseDateTime(date)
     DateTimeFormat.forPattern(format).print(currentDate.plusDays(1))
   }
@@ -283,17 +478,15 @@ object DateHelper extends Serializable {
   /** Returns for a date the date one day before.
     *
     * {{{
-    * // If the given date is "20170310":
     * assert(DateHelper.previousDay("20170310") == "20170309")
     * assert(DateHelper.previousDay("170310", "yyMMdd") == "170309")
     * }}}
     *
     * @param date the date for which to find the date of the day before
-    * @param format (default = "yyyyMMdd") the format of the provided and the
-    * returned dates.
+    * @param format the format of the provided and the returned dates
     * @return the date of the day before the given date
     */
-  def previousDay(date: String, format: String = "yyyyMMdd"): String = {
+  def previousDay(date: String, format: String = defaultFormat): String = {
     val currentDate = DateTimeFormat.forPattern(format).parseDateTime(date)
     DateTimeFormat.forPattern(format).print(currentDate.minusDays(1))
   }
@@ -307,10 +500,10 @@ object DateHelper extends Serializable {
     * }}}
     *
     * @param date the date for which to find the nbr of days of diff with today
-    * @param format (default = "yyyyMMdd") the format of the provided date
+    * @param format the format of the provided date
     * @return the nbr of days between today and the given date
     */
-  def nbrOfDaysSince(date: String, format: String = "yyyyMMdd"): Int =
+  def nbrOfDaysSince(date: String, format: String = defaultFormat): Int =
     Days
       .daysBetween(
         DateTimeFormat.forPattern(format).parseDateTime(date),
@@ -331,13 +524,13 @@ object DateHelper extends Serializable {
     * days.
     * @param lastDate the last date of the range for which to egt the nbr of
     * days.
-    * @param format (default = "yyyyMMdd") the format of the provided dates
+    * @param format the format of the provided dates
     * @return the nbr of days between the two given dates
     */
   def nbrOfDaysBetween(
       firstDate: String,
       lastDate: String,
-      format: String = "yyyyMMdd"
+      format: String = defaultFormat
   ): Int = {
 
     val formatter = DateTimeFormat.forPattern(format).withZone(DateTimeZone.UTC)
@@ -359,12 +552,12 @@ object DateHelper extends Serializable {
     *
     * @param timestamp the UTC timestamps (nbr of millis since 1970-01-01) for
     * which to get the associated date.
-    * @param format (default = "yyyyMMdd") the format of the provided dates
+    * @param format the format of the provided dates
     * @return the associated date under the requested format
     */
   def dateFromTimestamp(
       timestamp: Long,
-      format: String = "yyyyMMdd"
+      format: String = defaultFormat
   ): String =
     DateTimeFormat
       .forPattern(format)
@@ -377,11 +570,10 @@ object DateHelper extends Serializable {
     * {{{ assert(DateHelper.dayOfWeek("20160614") == 2) }}}
     *
     * @param date the date for which to get the day of week
-    * @param format (default = "yyyyMMdd") the format under which the date is
-    * provided.
+    * @param format the format under which the date is provided
     * @return the associated day of week, such as 2 for Tuesday
     */
-  def dayOfWeek(date: String, format: String = "yyyyMMdd"): Int =
+  def dayOfWeek(date: String, format: String = defaultFormat): Int =
     DateTimeFormat.forPattern(format).parseDateTime(date).getDayOfWeek()
 
   /** Validates a string date is under the provided format.
diff --git a/src/test/scala/com/spark_helper/DateHelperTest.scala b/src/test/scala/com/spark_helper/DateHelperTest.scala
index 6d27b81..30e3dd8 100644
--- a/src/test/scala/com/spark_helper/DateHelperTest.scala
+++ b/src/test/scala/com/spark_helper/DateHelperTest.scala
@@ -1,5 +1,7 @@
 package com.spark_helper
 
+import com.spark_helper.DateHelper._
+
 import org.scalatest.FunSuite
 
 import com.spark_helper.{DateHelper => DH}
@@ -85,6 +87,7 @@ class DateHelperTest extends FunSuite {
   }
 
   test("Date versus provided format") {
+
     assert(DateHelper.isDateCompliantWithFormat("20170302", "yyyyMMdd"))
     assert(!DateHelper.isDateCompliantWithFormat("20170333", "yyyyMMdd"))
     assert(DateHelper.isDateCompliantWithFormat("20170228", "yyyyMMdd"))
@@ -93,5 +96,14 @@ class DateHelperTest extends FunSuite {
     assert(!DateHelper.isDateCompliantWithFormat("", "yyyyMMdd"))
     assert(!DateHelper.isDateCompliantWithFormat("a", "yyyyMMdd"))
     assert(!DateHelper.isDateCompliantWithFormat("24JAN17", "yyyyMMdd"))
+
+    assert("20170302".isCompliantWith("yyyyMMdd"))
+    assert(!"20170333".isCompliantWith("yyyyMMdd"))
+    assert("20170228".isCompliantWith("yyyyMMdd"))
+    assert(!"20170229".isCompliantWith("yyyyMMdd"))
+    assert(!"170228".isCompliantWith("yyyyMMdd"))
+    assert(!"".isCompliantWith("yyyyMMdd"))
+    assert(!"a".isCompliantWith("yyyyMMdd"))
+    assert(!"24JAN17".isCompliantWith("yyyyMMdd"))
   }
 }

From 6d2868e38e011db51f7677a5583a276188c9cfca Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Thu, 14 Jun 2018 23:27:17 +0100
Subject: [PATCH 22/25] Add 20161230 to 20170101 method

---
 README.md                                     |  1 +
 .../scala/com/spark_helper/DateHelper.scala   | 20 +++++++++++++++----
 .../com/spark_helper/DateHelperTest.scala     |  6 +++++-
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 9ec5927..edbad1b 100644
--- a/README.md
+++ b/README.md
@@ -155,6 +155,7 @@ DateHelper.dayOfWeek("20160614") // 2
 import com.spark_helper.DateHelper._
 
 2.daysAgo // "20170308"
+"20161230" to "20170101" // List("20161230", "20161231", "20170101")
 3.daysBefore("20170310") // "20170307"
 5.daysAfter // "20170315"
 4.daysAfter("20170310") // "20170314"
diff --git a/src/main/scala/com/spark_helper/DateHelper.scala b/src/main/scala/com/spark_helper/DateHelper.scala
index 9d11627..fdb1885 100644
--- a/src/main/scala/com/spark_helper/DateHelper.scala
+++ b/src/main/scala/com/spark_helper/DateHelper.scala
@@ -28,7 +28,6 @@ import scala.util.Try
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/DateHelper.scala">DateHelper</a>
   *
-  * @todo Is it possible to ddo something like ("20170325" to "20170327")?
   * @author Xavier Guihot
   * @since 2017-02
   */
@@ -146,6 +145,19 @@ object DateHelper extends Serializable {
       * @return the date of the day before the given date
       */
     def previousDay(): String = DateHelper.previousDay(string)
+
+    /** Creates the list of dates between the two given dates.
+      *
+      * {{{
+      * assert(("20161230" to "20170101") == List("20161230", "20161231", "20170101"))
+      * }}}
+      *
+      * @param lastDate the last date
+      * @return the list of dates between this string and the lastDate in the
+      * default format.
+      */
+    def to(lastDate: String): List[String] =
+      DateHelper.daysBetween(string, lastDate)
   }
 
   /** Finds the list of dates between the two given dates.
@@ -156,15 +168,15 @@ object DateHelper extends Serializable {
     *
     * @param firstDate the first date (in the given format)
     * @param lastDate the last date (in the given format)
-    * @param format (default = "yyyyMMdd") the format to use for firstDate and
-    * lastDate and for the returned list of dates.
+    * @param format the format to use for firstDate and lastDate and for the
+    * returned list of dates.
     * @return the list of dates between firstDate and lastDate in the given
     * format.
     */
   def daysBetween(
       firstDate: String,
       lastDate: String,
-      format: String = "yyyyMMdd"
+      format: String = defaultFormat
   ): List[String] = {
 
     val formatter = DateTimeFormat.forPattern(format).withZone(DateTimeZone.UTC)
diff --git a/src/test/scala/com/spark_helper/DateHelperTest.scala b/src/test/scala/com/spark_helper/DateHelperTest.scala
index 30e3dd8..dd2a525 100644
--- a/src/test/scala/com/spark_helper/DateHelperTest.scala
+++ b/src/test/scala/com/spark_helper/DateHelperTest.scala
@@ -27,7 +27,11 @@ class DateHelperTest extends FunSuite {
     )
     assert(dates === expectedDates)
 
-    // 2: With a custom formatter:
+    // 2: Same as 1, but using the pimped String:
+    dates = "20161229" to "20170103"
+    assert(dates === expectedDates)
+
+    // 3: With a custom formatter:
     dates = DateHelper.daysBetween("29Dec16", "03Jan17", "ddMMMyy")
     expectedDates = List(
       "29Dec16",

From 114dde301502661044af9fada0dec2c5604bef57 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Fri, 15 Jun 2018 19:12:05 +0100
Subject: [PATCH 23/25] Minor refactoring - fix typos

---
 .gitignore                                    |  2 +
 README.md                                     | 33 ++++---
 .../scala/com/spark_helper/DateHelper.scala   | 30 +++---
 .../scala/com/spark_helper/HdfsHelper.scala   | 48 +++++-----
 src/main/scala/com/spark_helper/Monitor.scala | 93 +++++++++----------
 .../scala/com/spark_helper/SparkHelper.scala  | 65 +++++++------
 .../com/spark_helper/monitoring/Test.scala    | 12 +--
 .../org/apache/spark/TextFileOverwrite.scala  |  6 +-
 .../com/spark_helper/DateHelperTest.scala     |  2 +-
 .../com/spark_helper/HdfsHelperTest.scala     | 13 +--
 .../scala/com/spark_helper/MonitorTest.scala  | 77 +++++++--------
 .../com/spark_helper/SparkHelperTest.scala    | 67 ++++++-------
 12 files changed, 210 insertions(+), 238 deletions(-)

diff --git a/.gitignore b/.gitignore
index d838934..b76ffde 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@ project/target
 target
 
 *.crc
+
+.idea
diff --git a/README.md b/README.md
index edbad1b..26818eb 100644
--- a/README.md
+++ b/README.md
@@ -43,11 +43,11 @@ import com.spark_helper.HdfsHelper
 
 // A bunch of methods wrapping the FileSystem API, such as:
 HdfsHelper.fileExists("my/hdfs/file/path.txt") // HdfsHelper.folderExists("my/hdfs/folder")
-assert(HdfsHelper.listFileNamesInFolder("my/folder/path") == List("file_name_1.txt", "file_name_2.csv"))
-assert(HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") == "20170306")
-assert(HdfsHelper.nbrOfDaysSinceFileWasLastModified("my/hdfs/file/path.txt") == 3)
+HdfsHelper.listFileNamesInFolder("my/folder/path") // List("file_name_1.txt", "file_name_2.csv")
+HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") // "20170306"
+HdfsHelper.nbrOfDaysSinceFileWasLastModified("my/hdfs/file/path.txt") // 3
 HdfsHelper.deleteFile("my/hdfs/file/path.csv") // HdfsHelper.deleteFolder("my/hdfs/folder")
-HdfsHelper.moveFolder("my/hdfs/folder") // HdfsHelper.moveFile("my/hdfs/file.txt")
+HdfsHelper.moveFolder("old/path", "new/path") // HdfsHelper.moveFile("old/path.txt", "new/path.txt")
 HdfsHelper.createEmptyHdfsFile("/some/hdfs/file/path.token") // HdfsHelper.createFolder("my/hdfs/folder")
 
 // File content helpers:
@@ -103,14 +103,13 @@ rdd.saveAsSingleTextFile("/my/output/file/path.txt", classOf[BZip2Codec])
 // splitting the input with \n), it splits the file in records based on a custom
 // delimiter. This way, xml, json, yml or any multi-line record file format can
 // be used with Spark:
-sc.textFile("/my/input/folder/path", "---\n")
+sc.textFile("/my/input/folder/path", "---\n") // for a yml file for instance
 
 // Equivalent to rdd.flatMap(identity) for RDDs of Seqs or Options:
 rdd.flatten
 
-// Equivalent to sparkContext.textFile(), but for each line is tupled with its
-// file path:
-SparkHelper.textFileWithFileName("folder", sparkContext)
+// Equivalent to sc.textFile(), but for each line is tupled with its file path:
+sc.textFileWithFileName("/my/input/folder/path")
 // which produces:
 // RDD(("folder/file_1.txt", "record1fromfile1"), ("folder/file_1.txt", "record2fromfile1"),
 //    ("folder/file_2.txt", "record1fromfile2"), ...)
@@ -176,15 +175,15 @@ DateHelper.setFormat("ddMMMyy")
 The full list of methods is available at
 [Monitor](http://xavierguihot.com/spark_helper/#com.spark_helper.Monitor$)
 
-It's a simple logger/report which contains a report that one can update from
-the driver and a success state. The idea is to persist job executions logs and
-errors (and forget about grepping unreadable yarn logs).
+It's a simple logger/report which contains a report and a state that one can
+update from the driver. The idea is to persist job executions logs and errors
+(and forget about grepping unreadable yarn logs).
 
-It's designed for perdiodic spark jobs (handles storage and purge of logs) and
+It's designed for periodic spark jobs (handles storage and purge of logs) and
 provides a way to handle kpis validation.
 
 Logs are stored on the go which means one can have a direct real time access of
-the job logs/status and it's current state (which can overwise be a pain if it
+the job logs/status and it's current state (which can otherwise be a pain if it
 means going through yarn logs, or even for certain production environments going
 through additional layers of software logs to get to yarn logs).
 
@@ -198,9 +197,9 @@ the logger for a clean logging.
 This is a "driver-only" logger and is not intended at logging concurrent actions
 from executors.
 
-Produced reports can easily be inserted in a notification email whenerver the
+Produced reports can easily be inserted in a notification email whenever the
 job fails, which saves a lot of time to maintainers operating on heavy
-production environements.
+production environments.
 
 The produced persisted report is also a way for downstream jobs to know the
 status of their input data.
@@ -238,7 +237,7 @@ try {
     Monitor.error(e, "My pipeline descirption") // whatever unexpected error
 }
 
-if (Monitor.isSuccess()) {
+if (Monitor.isSuccess) {
   val doMore = "Let's do some more stuff!"
   Monitor.log("My second pipeline description: success")
 }
@@ -249,7 +248,7 @@ Monitor.store()
 
 // At the end of the job, if the job isn't successfull, you might want to
 // crash it (for instance to get a notification from your scheduler):
-if (!Monitor.isSuccess()) throw new Exception() // or send an email, or ...
+if (!Monitor.isSuccess) throw new Exception() // or send an email, or ...
 ```
 
 At any time during the job, logs can be accessed from file
diff --git a/src/main/scala/com/spark_helper/DateHelper.scala b/src/main/scala/com/spark_helper/DateHelper.scala
index fdb1885..d393718 100644
--- a/src/main/scala/com/spark_helper/DateHelper.scala
+++ b/src/main/scala/com/spark_helper/DateHelper.scala
@@ -12,7 +12,7 @@ import scala.util.Try
   * spark job and replace it with methods fully tested whose name is
   * self-explanatory/readable.
   *
-  * A few exemples:
+  * A few examples:
   *
   * {{{
   * assert(DateHelper.daysBetween("20161230", "20170101") == List("20161230", "20161231", "20170101"))
@@ -64,7 +64,7 @@ object DateHelper extends Serializable {
       *
       * @return today's date minus the given nbr of days
       */
-    def daysAgo(): String = DateHelper.nDaysBefore(int)
+    def daysAgo: String = DateHelper.nDaysBefore(int)
 
     /** Returns which date it was x days before the given date.
       *
@@ -92,7 +92,7 @@ object DateHelper extends Serializable {
       *
       * @return today's date plus the given nbr of days
       */
-    def daysAfter(): String = DateHelper.nDaysAfter(int)
+    def daysAfter: String = DateHelper.nDaysAfter(int)
 
     /** Returns which date it will be x days after the given date under the
       * default format.
@@ -112,7 +112,7 @@ object DateHelper extends Serializable {
 
   implicit class StringExtensions(val string: String) {
 
-    /** Validates the stringified date is compliant with the provided format.
+    /** Validates the formatted date is compliant with the provided format.
       *
       * {{{
       * assert("20170302".isCompliantWith("yyyyMMdd"))
@@ -136,7 +136,7 @@ object DateHelper extends Serializable {
       *
       * @return the date of the day after the given date
       */
-    def nextDay(): String = DateHelper.nextDay(string)
+    def nextDay: String = DateHelper.nextDay(string)
 
     /** Returns the date one day before the given date.
       *
@@ -144,7 +144,7 @@ object DateHelper extends Serializable {
       *
       * @return the date of the day before the given date
       */
-    def previousDay(): String = DateHelper.previousDay(string)
+    def previousDay: String = DateHelper.previousDay(string)
 
     /** Creates the list of dates between the two given dates.
       *
@@ -205,7 +205,7 @@ object DateHelper extends Serializable {
   ): List[DateTime] = {
 
     val nbrOfDaysWithinRange =
-      Days.daysBetween(jodaFirstDate, jodaLastDate).getDays()
+      Days.daysBetween(jodaFirstDate, jodaLastDate).getDays
 
     (0 to nbrOfDaysWithinRange).toList.map(jodaFirstDate.plusDays)
   }
@@ -385,7 +385,7 @@ object DateHelper extends Serializable {
     *
     * @return today's date under the default format
     */
-  def today(): String = nDaysBefore(0, defaultFormat)
+  def today: String = nDaysBefore(0, defaultFormat)
 
   /** Returns yesterday's date/time under the requested format.
     *
@@ -408,7 +408,7 @@ object DateHelper extends Serializable {
     *
     * @return yesterday's date under the default format
     */
-  def yesterday(): String = nDaysBefore(1, defaultFormat)
+  def yesterday: String = nDaysBefore(1, defaultFormat)
 
   /** Returns which date it was 2 days before today under the requested format.
     *
@@ -460,7 +460,7 @@ object DateHelper extends Serializable {
     * @return the current timestamps (nbr of millis since 1970-01-01) in the
     * local computer's zone.
     */
-  def currentTimestamp(): String = new DateTime().getMillis().toString
+  def currentTimestamp: String = new DateTime().getMillis.toString
 
   /** Returns the current UTC timestamp.
     *
@@ -469,7 +469,7 @@ object DateHelper extends Serializable {
     * @return the current UTC timestamps (nbr of millis since 1970-01-01).
     */
   def currentUtcTimestamp(): String =
-    new DateTime().withZone(DateTimeZone.UTC).getMillis().toString
+    new DateTime().withZone(DateTimeZone.UTC).getMillis.toString
 
   /** Returns for a date the date one day latter.
     *
@@ -521,7 +521,7 @@ object DateHelper extends Serializable {
         DateTimeFormat.forPattern(format).parseDateTime(date),
         new DateTime()
       )
-      .getDays()
+      .getDays
 
   /** Returns the nbr of days between the two given dates.
     *
@@ -552,7 +552,7 @@ object DateHelper extends Serializable {
         formatter.parseDateTime(firstDate),
         formatter.parseDateTime(lastDate)
       )
-      .getDays()
+      .getDays
   }
 
   /** Returns the date associated to the given UTC timestamp.
@@ -586,7 +586,7 @@ object DateHelper extends Serializable {
     * @return the associated day of week, such as 2 for Tuesday
     */
   def dayOfWeek(date: String, format: String = defaultFormat): Int =
-    DateTimeFormat.forPattern(format).parseDateTime(date).getDayOfWeek()
+    DateTimeFormat.forPattern(format).parseDateTime(date).getDayOfWeek
 
   /** Validates a string date is under the provided format.
     *
@@ -601,7 +601,7 @@ object DateHelper extends Serializable {
     * assert(!DateHelper.isDateCompliantWithFormat("24JAN17", "yyyyMMdd"))
     * }}}
     *
-    * @param stringValue the stringified date
+    * @param stringValue the formatted date
     * @return if the provided date is under the provided format
     */
   def isDateCompliantWithFormat(
diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index 920c905..b4cd82a 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -36,7 +36,7 @@ import com.typesafe.config.{Config, ConfigFactory}
   * code and thus could instead just use
   * HdfsHelper.deleteFile("my/hdfs/file/path.csv").
   *
-  * A few exemples:
+  * A few examples:
   *
   * {{{
   * import com.spark_helper.HdfsHelper
@@ -49,7 +49,7 @@ import com.typesafe.config.{Config, ConfigFactory}
   * HdfsHelper.deleteFile("my/hdfs/file/path.csv")
   * HdfsHelper.moveFolder("my/hdfs/folder")
   * HdfsHelper.compressFile("hdfs/path/to/uncompressed_file.txt", classOf[GzipCodec])
-  * HdfsHelper.appendHeader("my/hdfs/file/path.csv", "colum0,column1")
+  * HdfsHelper.appendHeader("my/hdfs/file/path.csv", "column0,column1")
   *
   * // Some Xml/Typesafe helpers for hadoop as well:
   * HdfsHelper.isHdfsXmlCompliantWithXsd(
@@ -57,7 +57,7 @@ import com.typesafe.config.{Config, ConfigFactory}
   * HdfsHelper.loadXmlFileFromHdfs("my/hdfs/file/path.xml")
   *
   * // Very handy to load a config (typesafe format) stored on hdfs at the
-  * // begining of a spark job:
+  * // beginning of a spark job:
   * HdfsHelper.loadTypesafeConfigFromHdfs("my/hdfs/file/path.conf"): Config
   *
   * // In order to write small amount of data in a file on hdfs without the
@@ -90,7 +90,7 @@ object HdfsHelper extends Serializable {
     * If this setter is not used, the default Configuration is set with
     * <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">new Configuration()</code>.
     *
-    * @param conf the specific Configuration to use
+    * @param configuration the specific Configuration to use
     */
   def setConf(configuration: Configuration): Unit = {
     conf = configuration
@@ -306,8 +306,8 @@ object HdfsHelper extends Serializable {
 
   /** Creates an empty file on hdfs.
     *
-    * Might be usefull for token files. For instance a file which is only used
-    * as a timestamp token of the last update of a processus, or a file which
+    * Might be useful for token files. For instance a file which is only used
+    * as a timestamp token of the last update of a process, or a file which
     * blocks the execution of an other instance of the same job, ...
     *
     * Overwrites the file if it already exists.
@@ -398,7 +398,7 @@ object HdfsHelper extends Serializable {
         else if (recursive)
           listFileNamesInFolder(
             hdfsPath + "/" + status.getPath.getName,
-            true,
+            recursive = true,
             onlyName
           )
         // If it's a dir and we're not in a recursive option:
@@ -433,9 +433,9 @@ object HdfsHelper extends Serializable {
     * @return the joda DateTime of the last modification of the given file
     */
   def fileModificationDateTime(hdfsPath: String): DateTime =
-    new DateTime(hdfs.getFileStatus(new Path(hdfsPath)).getModificationTime())
+    new DateTime(hdfs.getFileStatus(new Path(hdfsPath)).getModificationTime)
 
-  /** Returns the stringified date of the last modification of the given file.
+  /** Returns the formatted date of the last modification of the given file.
     *
     * {{{
     * assert(HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") == "20170306")
@@ -445,7 +445,7 @@ object HdfsHelper extends Serializable {
     * modification date.
     * @param format (default = "yyyyMMdd") the format under which to get the
     * modification date.
-    * @return the stringified date of the last modification of the given file,
+    * @return the formatted date of the last modification of the given file,
     * under the provided format.
     */
   def fileModificationDate(
@@ -463,7 +463,7 @@ object HdfsHelper extends Serializable {
   def folderModificationDateTime(hdfsPath: String): DateTime =
     fileModificationDateTime(hdfsPath)
 
-  /** Returns the stringified date of the last modification of the given folder.
+  /** Returns the formatted date of the last modification of the given folder.
     *
     * {{{
     * assert(HdfsHelper.folderModificationDate("my/hdfs/folder") == "20170306")
@@ -473,7 +473,7 @@ object HdfsHelper extends Serializable {
     * modification date.
     * @param format (default = "yyyyMMdd") the format under which to get the
     * modification date.
-    * @return the stringified date of the last modification of the given folder,
+    * @return the formatted date of the last modification of the given folder,
     * under the provided format.
     */
   def folderModificationDate(
@@ -495,17 +495,17 @@ object HdfsHelper extends Serializable {
   def nbrOfDaysSinceFileWasLastModified(hdfsPath: String): Int =
     Days
       .daysBetween(fileModificationDateTime(hdfsPath), new DateTime())
-      .getDays()
+      .getDays
 
   /** Appends a header and a footer to a file.
     *
-    * Usefull when creating an xml file with spark and you need to add top level
+    * Useful when creating an xml file with spark and you need to add top level
     * tags.
     *
     * If the workingFolderPath parameter is provided, then the processing is
     * done in a working/tmp folder and then only, the final file is moved to its
     * final real location. This way, in case of cluster instability, i.e. in
-    * case the Spark job is interupted, this avoids having a temporary or
+    * case the Spark job is interrupted, this avoids having a temporary or
     * corrupted file in output.
     *
     * @param filePath the path of the file for which to add the header and the
@@ -528,13 +528,13 @@ object HdfsHelper extends Serializable {
 
   /** Appends a header to a file.
     *
-    * Usefull when creating a csv file with spark and you need to add a header
+    * Useful when creating a csv file with spark and you need to add a header
     * describing the different fields.
     *
     * If the workingFolderPath parameter is provided, then the processing is
     * done in a working/tmp folder and then only, the final file is moved to its
     * final real location. This way, in case of cluster instability, i.e. in
-    * case the Spark job is interupted, this avoids having a temporary or
+    * case the Spark job is interrupted, this avoids having a temporary or
     * corrupted file in output.
     *
     * @param filePath the path of the file for which to add the header
@@ -557,7 +557,7 @@ object HdfsHelper extends Serializable {
     * If the workingFolderPath parameter is provided, then the processing is
     * done in a working/tmp folder and then only, the final file is moved to its
     * final real location. This way, in case of cluster instability, i.e. in
-    * case the Spark job is interupted, this avoids having a temporary or
+    * case the Spark job is interrupted, this avoids having a temporary or
     * corrupted file in output.
     *
     * @param filePath the path of the file for which to add the footer
@@ -589,7 +589,7 @@ object HdfsHelper extends Serializable {
       validateHdfsXmlWithXsd(hdfsXmlPath, xsdFile)
       true
     } catch {
-      case saxe: SAXException => false
+      case _: SAXException => false
     }
 
   /** Validates an XML file on hdfs in regard to the given XSD.
@@ -615,7 +615,7 @@ object HdfsHelper extends Serializable {
     validator.validate(xmlFile)
   }
 
-  /** Loads a typesafe config from Hdfs.
+  /** Loads a Typesafe config from Hdfs.
     *
     * The best way to load the configuration of your job from hdfs.
     *
@@ -643,8 +643,8 @@ object HdfsHelper extends Serializable {
     * }
     * }}}
     *
-    * @param hdfsConfigPath the absolute path of the typesafe config file on
-    * hdfs we want to load as a typesafe Config object.
+    * @param hdfsConfigPath the absolute path of the Typesafe config file on
+    * hdfs we want to load as a Typesafe Config object.
     * @return the com.typesafe.config.Config object which contains usable data
     */
   def loadTypesafeConfigFromHdfs(hdfsConfigPath: String): Config = {
@@ -746,8 +746,8 @@ object HdfsHelper extends Serializable {
       .filter(path => {
 
         val fileAgeInDays = Days
-          .daysBetween(new DateTime(path.getModificationTime()), new DateTime())
-          .getDays()
+          .daysBetween(new DateTime(path.getModificationTime), new DateTime())
+          .getDays
 
         fileAgeInDays >= purgeAge
 
diff --git a/src/main/scala/com/spark_helper/Monitor.scala b/src/main/scala/com/spark_helper/Monitor.scala
index 3866cbb..f5eeeca 100644
--- a/src/main/scala/com/spark_helper/Monitor.scala
+++ b/src/main/scala/com/spark_helper/Monitor.scala
@@ -6,19 +6,17 @@ import java.util.Calendar
 
 import org.apache.commons.lang3.time.DurationFormatUtils
 
-import java.lang.Throwable
-
-/** A logger dedicated to Spak jobs.
+/** A logger dedicated to Spark jobs.
   *
   * It's a simple logger/report which contains a report that one can update from
   * the driver and a success state. The idea is to persist job executions logs
   * and errors (and forget about grepping unreadable yarn logs).
   *
-  * It's designed for perdiodic spark jobs (handles storage and purge of logs)
+  * It's designed for periodic spark jobs (handles storage and purge of logs)
   * and provides a way to handle kpis validation.
   *
   * Logs are stored on the go which means one can have a direct real time access
-  * of the job logs/status and it's current state (which can overwise be a pain
+  * of the job logs/status and it's current state (which can otherwise be a pain
   * if it means going through yarn logs, or even for certain production
   * environments going through additional layers of software logs to get to yarn
   * logs).
@@ -33,9 +31,9 @@ import java.lang.Throwable
   * This is a "driver-only" logger and is not intended at logging concurrent
   * actions from executors.
   *
-  * Produced reports can easily be inserted in a notification email whenerver
+  * Produced reports can easily be inserted in a notification email whenever
   * the job fails, which saves a lot of time to maintainers operating on heavy
-  * production environements.
+  * production environments.
   *
   * The produced persisted report is also a way for downstream jobs to know the
   * status of their input data.
@@ -61,7 +59,7 @@ import java.lang.Throwable
   *       Test("Nbr of output records", processedData.count(), SUPERIOR_THAN, 10e6d, NBR),
   *       Test("Some pct of invalid output", your_complex_kpi, INFERIOR_THAN, 3, PCT)
   *     ),
-  *     "My pipeline descirption"
+  *     "My pipeline description"
   *   )
   *
   *   if (outputIsValid)
@@ -69,9 +67,9 @@ import java.lang.Throwable
   *
   * } catch {
   *   case iie: InvalidInputException =>
-  *     Monitor.error(iie, "My pipeline descirption", diagnostic = "No input data!")
+  *     Monitor.error(iie, "My pipeline description", diagnostic = "No input data!")
   *   case e: Throwable =>
-  *     Monitor.error(e, "My pipeline descirption") // whatever unexpected error
+  *     Monitor.error(e, "My pipeline description") // whatever unexpected error
   * }
   *
   * if (Monitor.isSuccess()) {
@@ -83,7 +81,7 @@ import java.lang.Throwable
   * // HDFS (this saves the logs in the folder set with Monitor.setLogFolder):
   * Monitor.store()
   *
-  * // At the end of the job, if the job isn't successfull, you might want to
+  * // At the end of the job, if the job isn't successful, you might want to
   * // crash it (for instance to get a notification from your scheduler):
   * if (!Monitor.isSuccess()) throw new Exception() // or send an email, or ...
   * }}}
@@ -100,8 +98,8 @@ import java.lang.Throwable
   *
   * My job description (whatever you want); for instance:
   * Documentation: https://github.com/xavierguihot/spark_helper
-  * [10:23] Begining
-  * [10:23-10:23] My pipeline descirption: failed
+  * [10:23] Beginning
+  * [10:23-10:23] My pipeline description: failed
   *   Diagnostic: No input data!
   *     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs://my/hdfs/input/path
   *     at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:285)
@@ -116,8 +114,8 @@ import java.lang.Throwable
   *
   * My job description (whatever you want); for instance:
   * Documentation: https://github.com/xavierguihot/spark_helper
-  * [10:23] Begining
-  * [10:23-10:36] My pipeline descirption: failed
+  * [10:23] Beginning
+  * [10:23-10:36] My pipeline description: failed
   *     java.lang.NumberFormatException: For input string: "a"
   *     java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
   *     java.lang.Integer.parseInt(Integer.java:492)
@@ -125,14 +123,14 @@ import java.lang.Throwable
   * [10:36] Duration: 00:13:47
   * }}}
   *
-  * Another scenario, successfull spark pipeline and KPIs are valid; all good!:
+  * Another scenario, successful spark pipeline and KPIs are valid; all good!:
   * {{{
   *           My job title
   *
   * My job description (whatever you want); for instance:
   * Documentation: https://github.com/xavierguihot/spark_helper
-  * [10:23] Begining
-  * [10:23-10:41] My pipeline descirption: success
+  * [10:23] Beginning
+  * [10:23-10:41] My pipeline description: success
   *   KPI: Nbr of output records
   *     Value: 14669071.0
   *     Must be superior than 10000000.0
@@ -148,6 +146,7 @@ import java.lang.Throwable
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
   * /main/scala/com/spark_helper/monitoring/Monitor.scala">Monitor</a>
   *
+  * @todo would a State monad be appropriate?
   * @author Xavier Guihot
   * @since 2017-02
   */
@@ -159,13 +158,13 @@ object Monitor {
   private var logDirectory: Option[String] = None
   private var purgeWindow: Option[Int] = None
 
-  private val jobStart = DateHelper.now("[HH:mm]") + " Begining"
+  private val jobStart = DateHelper.now("[HH:mm]") + " Beginning"
 
   // Join of reportTitle, pointsOfContact, reportDescription, logDirectory and
   // jobStart:
   private var reportHeader = buildReportHeader()
 
-  private val begining = Calendar.getInstance().getTimeInMillis()
+  private val beginning = Calendar.getInstance().getTimeInMillis
   private var lastReportUpdate = DateHelper.now("HH:mm")
 
   /** Sets the report's title.
@@ -175,7 +174,7 @@ object Monitor {
     * {{{
     * // Using:
     * Monitor.setReportTitle("My Simple Job")
-    * // Produces this at the begining of the report:
+    * // Produces this at the beginning of the report:
     * "          My Simple Job"
     * ""
     * }}}
@@ -196,7 +195,7 @@ object Monitor {
     * // Using:
     * Monitor.setReportTitle("My Simple Job")
     * Monitor.addContacts(List("x.guihot@gmail.com", "smbdy@gmail.com"))
-    * // Produces this at the begining of the report:
+    * // Produces this at the beginning of the report:
     * "          My Simple Job"
     * ""
     * "Point of contact: x.guihot@gmail.com, smbdy@gmail.com"
@@ -218,7 +217,7 @@ object Monitor {
     * // Using:
     * Monitor.setReportTitle("My Simple Job")
     * Monitor.addDescription("Documentation: https://github.com/xavierguihot/spark_helper")
-    * // Produces this at the begining of the report:
+    * // Produces this at the beginning of the report:
     * "          My Simple Job"
     * ""
     * "Documentation: https://github.com/xavierguihot/spark_helper"
@@ -269,7 +268,7 @@ object Monitor {
     *
     * @return if your spark job is successful.
     */
-  def isSuccess(): Boolean = successful
+  def isSuccess: Boolean = successful
 
   /** Returns the current state of the monitoring report.
     *
@@ -286,7 +285,7 @@ object Monitor {
     *
     * @param text the text to append to the report
     */
-  def log(text: String): Unit = log(text, true)
+  def log(text: String): Unit = log(text, withTimestamp = true)
 
   /** Updates the report with some text and a success.
     *
@@ -316,7 +315,7 @@ object Monitor {
     * will result in this to be appended to the report:
     * {{{ "[10:35-10:37] Some text: failure\n" }}}
     *
-    * Once the monitoring is a failure, then whatever following successfull
+    * Once the monitoring is a failure, then whatever following successful
     * action won't change the failed status of the monitoring.
     *
     * @param taskDescription the text to append to the report
@@ -342,12 +341,12 @@ object Monitor {
     * {{{
     * monitor.error(
     *   invalidInputException,
-    *   "My pipeline descirption",
+    *   "My pipeline description",
     *   diagnostic = "No input data!")
     * }}}
     * will result in this to be appended to the report:
     * {{{
-    * [10:23-10:24] My pipeline descirption: failed
+    * [10:23-10:24] My pipeline description: failed
     *   Diagnostic: No input data!
     *     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs://my/hdfs/input/path
     *     at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:285)
@@ -370,7 +369,7 @@ object Monitor {
     successful = false
 
     val serializedException =
-      "\t\t" + exception.toString() + "\n" +
+      "\t\t" + exception.toString + "\n" +
         exception.getStackTrace.map(line => s"\t\t$line").mkString("\n")
 
     val update = List(
@@ -433,14 +432,13 @@ object Monitor {
     if (!testsAreValid)
       successful = false
 
-    val seriralizedTests = tests.mkString("\n")
+    val serializedTests = tests.mkString("\n")
 
     val update = testSuitName match {
-      case "" => seriralizedTests
-      case _ => {
+      case "" => serializedTests
+      case _ =>
         val status = if (testsAreValid) "success" else "failed"
-        s"$testSuitName: $status\n$seriralizedTests"
-      }
+        s"$testSuitName: $status\n$serializedTests"
     }
 
     log(update)
@@ -506,11 +504,10 @@ object Monitor {
 
     logDirectory match {
 
-      case Some(logFolder) => {
-
+      case Some(logFolder) =>
         // We add the job duration to the report:
         val jobDuration = DurationFormatUtils.formatDuration(
-          Calendar.getInstance().getTimeInMillis() - begining,
+          Calendar.getInstance().getTimeInMillis - beginning,
           "HH:mm:ss")
 
         var now = DateHelper.now("[HH:mm]")
@@ -532,7 +529,6 @@ object Monitor {
           .writeToHdfsFile(finalReport, s"$logFolder/current.$reportExtension")
 
         purgeWindow.foreach(window => purgeOutdatedLogs(logFolder, window))
-      }
 
       case None =>
         require(
@@ -583,20 +579,17 @@ object Monitor {
   /** Updates the current stored version of logs in file
     * logFolder/current.ongoing */
   private def storeCurrent(): Unit =
-    logDirectory.foreach {
-      case logFolder => {
-
-        val warning =
-          "WARNING: If this file exists it does not necessarily mean that " +
-            "your job is still running. This file might persist if your job " +
-            "has been killed and thus couldn't reach your call to the " +
-            "Monitor.store()."
+    logDirectory.foreach { logFolder =>
+      val warning =
+        "WARNING: If this file exists it does not necessarily mean that " +
+          "your job is still running. This file might persist if your job " +
+          "has been killed and thus couldn't reach your call to the " +
+          "Monitor.store()."
 
-        val ongoingReport =
-          s"$reportHeader\n$report\n$warning"
+      val ongoingReport =
+        s"$reportHeader\n$report\n$warning"
 
-        HdfsHelper.writeToHdfsFile(ongoingReport, s"$logFolder/current.ongoing")
-      }
+      HdfsHelper.writeToHdfsFile(ongoingReport, s"$logFolder/current.ongoing")
     }
 
   private def purgeOutdatedLogs(logFolder: String, window: Int): Unit = {
diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index 336ff4a..f87edf9 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -21,7 +21,7 @@ import scala.util.Random
   * spark job and replace it with methods fully tested whose name is
   * self-explanatory/readable.
   *
-  * A few exemples:
+  * A few examples:
   *
   * {{{
   * // Same as sc.saveAsTextFile(path), but the result is a single file:
@@ -83,7 +83,7 @@ object SparkHelper extends Serializable {
     /** Saves an RDD in exactly one file.
       *
       * Allows one to save an RDD in one file, while keeping the processing
-      * parallelized.
+      * distributed.
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt") }}}
       *
@@ -95,7 +95,7 @@ object SparkHelper extends Serializable {
     /** Saves an RDD in exactly one file.
       *
       * Allows one to save an RDD in one file, while keeping the processing
-      * parallelized.
+      * distributed.
       *
       * {{{ rdd.saveAsSingleTextFile("/my/file/path.txt", classOf[BZip2Codec]) }}}
       *
@@ -112,7 +112,7 @@ object SparkHelper extends Serializable {
     /** Saves an RDD in exactly one file.
       *
       * Allows one to save an RDD in one file, while keeping the processing
-      * parallelized.
+      * distributed.
       *
       * This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
       * performs the storage in a temporary folder instead of directly in the
@@ -136,7 +136,7 @@ object SparkHelper extends Serializable {
     /** Saves an RDD in exactly one file.
       *
       * Allows one to save an RDD in one file, while keeping the processing
-      * parallelized.
+      * distributed.
       *
       * This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
       * performs the storage in a temporary folder instead of directly in the
@@ -174,8 +174,8 @@ object SparkHelper extends Serializable {
       * The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
       * , but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
       * is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
-      * would make the processing time explode, wherease this methods keeps the
-      * processing parallelized, save as text file and then only merges the
+      * would make the processing time explode, whereas this methods keeps the
+      * processing distributed, save as text file and then only merges the
       * result in a lower nbr of partitions.
       *
       * {{{ rdd.saveAsTextFileAndCoalesce("/produced/folder/path/with/only/30/files", 30) }}}
@@ -219,8 +219,8 @@ object SparkHelper extends Serializable {
       * The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
       * , but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
       * is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
-      * would make the processing time explode, wherease this methods keeps the
-      * processing parallelized, save as text file and then only merges the
+      * would make the processing time explode, whereas this methods keeps the
+      * processing distributed, save as text file and then only merges the
       * result in a lower nbr of partitions.
       *
       * {{{ rdd.saveAsTextFileAndCoalesce("/produced/folder/path/with/only/30/files", 30, classOf[BZip2Codec]) }}}
@@ -261,7 +261,7 @@ object SparkHelper extends Serializable {
 
   implicit class SeqRDDExtensions[T: ClassTag](val rdd: RDD[Seq[T]]) {
 
-    /** Flattens an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[Seq[T]]</code>
+    /** Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Seq[T]</code>
       * to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.
       *
       * {{{ sc.parallelize(Array(Seq(1, 2, 3), Nil, Seq(4))).flatten == sc.parallelize(Array(Seq(1, 2, 3, 4))) }}}
@@ -270,12 +270,12 @@ object SparkHelper extends Serializable {
       * or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
       * would have.
       */
-    def flatten(): RDD[T] = rdd.flatMap(identity)
+    def flatten: RDD[T] = rdd.flatMap(identity)
   }
 
   implicit class OptionRDDExtensions[T: ClassTag](val rdd: RDD[Option[T]]) {
 
-    /** Flattens an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[Option[T]]</code>
+    /** Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Option[T]</code>
       * to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.
       *
       * {{{ sc.parallelize(Array(Some(1), None, Some(2))).flatten == sc.parallelize(Array(Seq(1, 2))) }}}
@@ -284,7 +284,7 @@ object SparkHelper extends Serializable {
       * or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
       * would have.
       */
-    def flatten(): RDD[T] = rdd.flatMap(o => o)
+    def flatten: RDD[T] = rdd.flatMap(o => o)
   }
 
   implicit class PairRDDExtensions(val rdd: RDD[(String, String)])
@@ -306,7 +306,7 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path") }}}
       *
-      * @param path the folder where will be storrred key files
+      * @param path the folder where will be stored key files
       */
     def saveAsTextFileByKey(path: String): Unit =
       SparkHelper.saveAsTextFileByKeyInternal(rdd, path, None, None)
@@ -322,8 +322,8 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", 12) }}}
       *
-      * @param path the folder where will be storrred key files
-      * @param keyNbr the nbr of expected keys (which is the nbr of outputed
+      * @param path the folder where will be stored key files
+      * @param keyNbr the nbr of expected keys (which is the nbr of output
       * files)
       */
     def saveAsTextFileByKey(path: String, keyNbr: Int): Unit =
@@ -346,7 +346,7 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", classOf[BZip2Codec]) }}}
       *
-      * @param path the folder where will be storrred key files
+      * @param path the folder where will be stored key files
       * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
       */
@@ -367,8 +367,8 @@ object SparkHelper extends Serializable {
       *
       * {{{ rdd.saveAsTextFileByKey("/my/output/folder/path", 12, classOf[BZip2Codec]) }}}
       *
-      * @param path the folder where will be storrred key files
-      * @param keyNbr the nbr of expected keys (which is the nbr of outputed
+      * @param path the folder where will be stored key files
+      * @param keyNbr the nbr of expected keys (which is the nbr of output
       * files)
       * @param codec the type of compression to use (for instance
       * classOf[BZip2Codec] or classOf[GzipCodec]))
@@ -667,11 +667,11 @@ object SparkHelper extends Serializable {
   /** Saves RDD in exactly one file.
     *
     * Allows one to save an RDD as one text file, but at the same time to keep
-    * the processing parallelized.
+    * the processing distributed.
     *
     * @param outputRDD the RDD of strings to save as text file
     * @param path the path where to save the file
-    * @param compression the compression codec to use (can be left to None)
+    * @param codec the compression codec to use (can be left to None)
     */
   private def saveAsSingleTextFileInternal(
       outputRDD: RDD[String],
@@ -685,8 +685,8 @@ object SparkHelper extends Serializable {
     // Classic saveAsTextFile in a temporary folder:
     HdfsHelper.deleteFolder(s"$path.tmp")
     codec match {
-      case Some(codec) =>
-        outputRDD.saveAsTextFile(s"$path.tmp", codec)
+      case Some(compression) =>
+        outputRDD.saveAsTextFile(s"$path.tmp", compression)
       case None =>
         outputRDD.saveAsTextFile(s"$path.tmp")
     }
@@ -718,25 +718,22 @@ object SparkHelper extends Serializable {
     val isCached = rdd.getStorageLevel.useMemory
 
     // If the nbr of keys isn't provided, we have to get it ourselves:
-    val keyNbr = optKeyNbr match {
-      case Some(keyNbr) =>
-        keyNbr
-      case None =>
-        if (!isCached)
-          rdd.cache()
-        rdd.keys.distinct.count.toInt
+    val keyNbr = optKeyNbr.getOrElse {
+      if (!isCached)
+        rdd.cache()
+      rdd.keys.distinct.count.toInt
     }
 
     val prdd = rdd.partitionBy(new HashPartitioner(keyNbr))
 
     codec match {
-      case Some(codec) =>
+      case Some(compression) =>
         prdd.saveAsHadoopFile(
           path,
           classOf[String],
           classOf[String],
           classOf[KeyBasedOutput],
-          codec
+          compression
         )
       case None =>
         prdd.saveAsHadoopFile(
@@ -764,8 +761,8 @@ object SparkHelper extends Serializable {
       .coalesce(finalCoalesceLevel)
 
     codec match {
-      case Some(codec) =>
-        intermediateRDD.saveAsTextFile(lowerCoalescenceLevelFolder, codec)
+      case Some(compression) =>
+        intermediateRDD.saveAsTextFile(lowerCoalescenceLevelFolder, compression)
       case None =>
         intermediateRDD.saveAsTextFile(lowerCoalescenceLevelFolder)
     }
diff --git a/src/main/scala/com/spark_helper/monitoring/Test.scala b/src/main/scala/com/spark_helper/monitoring/Test.scala
index 97942e2..80b3ad7 100644
--- a/src/main/scala/com/spark_helper/monitoring/Test.scala
+++ b/src/main/scala/com/spark_helper/monitoring/Test.scala
@@ -7,7 +7,7 @@ import java.lang.Math.abs
   * This is intended to be used as parameter of Monitor.updateByKpiValidation
   * and Monitor.updateByKpisValidation methods.
   *
-  * Some exemples of Test objects:
+  * Some examples of Test objects:
   * {{{
   * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
   * Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
@@ -19,7 +19,7 @@ import java.lang.Math.abs
   *
   * @constructor Creates a Test object.
   *
-  * Some exemples of Test objects:
+  * Some examples of Test objects:
   * {{{
   * Test("pctOfWhatever", 0.06d, INFERIOR_THAN, 0.1d, PCT)
   * Test("pctOfSomethingElse", 0.27d, SUPERIOR_THAN, 0.3d, PCT)
@@ -42,22 +42,22 @@ final case class Test(
     kpiType: KpiType
 ) {
 
-  private[spark_helper] def isSuccess(): Boolean = thresholdType match {
+  private[spark_helper] def isSuccess: Boolean = thresholdType match {
     case EQUAL_TO      => kpiValue == appliedThreshold
     case SUPERIOR_THAN => abs(kpiValue) >= appliedThreshold
     case INFERIOR_THAN => abs(kpiValue) <= appliedThreshold
   }
 
-  override def toString(): String =
+  override def toString: String =
     List(
       "\tKPI: " + description,
       "\t\tValue: " + kpiValue.toString + kpiType.name,
       "\t\tMust be " + thresholdType.name + " " + appliedThreshold.toString + kpiType.name,
-      "\t\tValidated: " + isSuccess().toString
+      "\t\tValidated: " + isSuccess.toString
     ).mkString("\n")
 }
 
-/** An enumeration which represents the type of threshol to use (EQUAL_TO,
+/** An enumeration which represents the type of threshold to use (EQUAL_TO,
   * SUPERIOR_THAN or INFERIOR_THAN) */
 sealed trait ThresholdType { def name: String }
 
diff --git a/src/main/scala/org/apache/spark/TextFileOverwrite.scala b/src/main/scala/org/apache/spark/TextFileOverwrite.scala
index 8d1cc1b..28935ea 100644
--- a/src/main/scala/org/apache/spark/TextFileOverwrite.scala
+++ b/src/main/scala/org/apache/spark/TextFileOverwrite.scala
@@ -29,9 +29,9 @@ object TextFileOverwrite {
      * version of hadoop-common used by Spark (it will become Serializable
      * starting version 3 of hadoop-common).
      *
-     * * I don't String* (instead of Seq[String]) as for 1 String only it would
-     * confuse the compiler as to which sc.textFile to use (the default one or
-     * this one).
+     * * I don't use String* (instead of Seq[String]) as for 1 String only it
+     * would confuse the compiler as to which sc.textFile to use (the default
+     * one or this one).
      */
 
     val confBroadcast =
diff --git a/src/test/scala/com/spark_helper/DateHelperTest.scala b/src/test/scala/com/spark_helper/DateHelperTest.scala
index dd2a525..5c68404 100644
--- a/src/test/scala/com/spark_helper/DateHelperTest.scala
+++ b/src/test/scala/com/spark_helper/DateHelperTest.scala
@@ -81,7 +81,7 @@ class DateHelperTest extends FunSuite {
     assert(DateHelper.nDaysBeforeDate(5, "170310", "yyMMdd") === "170305")
   }
 
-  test("Date it will be N days affter date") {
+  test("Date it will be N days after date") {
     assert(DateHelper.nDaysAfterDate(3, "20170307") === "20170310")
     assert(DateHelper.nDaysAfterDate(5, "170305", "yyMMdd") === "170310")
   }
diff --git a/src/test/scala/com/spark_helper/HdfsHelperTest.scala b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
index f96db97..b64306d 100644
--- a/src/test/scala/com/spark_helper/HdfsHelperTest.scala
+++ b/src/test/scala/com/spark_helper/HdfsHelperTest.scala
@@ -158,7 +158,8 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
 
     // 3: Using the pimped Seq/String:
 
-    listToStore.toSeq.writeToHdfs(filePath)
+    val seqToStore = Seq("Hello World", "Whatever")
+    seqToStore.writeToHdfs(filePath)
     assert(HdfsHelper.fileExists(filePath))
     storedContent = sc.textFile(filePath).collect().sorted.mkString("\n")
     assert(storedContent === contentToStore)
@@ -281,7 +282,7 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     assert(HdfsHelper.fileExists(filePath))
     assert(!HdfsHelper.fileExists(renamedPath))
 
-    // 3: Let's successfuly move the file with the moveFile() method:
+    // 3: Let's successfully move the file with the moveFile() method:
 
     // Let's rename the file:
     HdfsHelper.moveFile(filePath, renamedPath)
@@ -326,7 +327,7 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     assert(HdfsHelper.fileExists(s"$folderToMove/file_2.txt"))
     assert(!HdfsHelper.folderExists(renamedFolder))
 
-    // 2: Let's successfuly move the folder with the moveFolder() method:
+    // 2: Let's successfully move the folder with the moveFolder() method:
 
     // Let's rename the folder:
     HdfsHelper.moveFolder(folderToMove, renamedFolder)
@@ -411,7 +412,7 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     HdfsHelper.deleteFolder(testFolder)
     HdfsHelper.writeToHdfsFile(
       "<Customer>\n" +
-        "	<Age>trente</Age>\n" +
+        "	<Age>thirty</Age>\n" +
         "	<Address>34 thingy street, someplace, sometown</Address>\n" +
         "</Customer>",
       xmlPath
@@ -500,11 +501,11 @@ class HdfsHelperTest extends FunSuite with SharedSparkContext {
     HdfsHelper.deleteFile(filePath)
 
     HdfsHelper.writeToHdfsFile("hello\nworld", filePath)
-    HdfsHelper.compressFile(filePath, classOf[GzipCodec], true)
+    HdfsHelper.compressFile(filePath, classOf[GzipCodec])
 
     assert(HdfsHelper.fileExists(s"$filePath.gz"))
 
-    // Easy to test with spark, as reading a file with the ".gz" extention
+    // Easy to test with spark, as reading a file with the ".gz" extension
     // forces the read with the compression codec:
     val content = sc.textFile(s"$filePath.gz").collect.sorted
     assert(content === Array("hello", "world"))
diff --git a/src/test/scala/com/spark_helper/MonitorTest.scala b/src/test/scala/com/spark_helper/MonitorTest.scala
index 45381ac..bda85da 100644
--- a/src/test/scala/com/spark_helper/MonitorTest.scala
+++ b/src/test/scala/com/spark_helper/MonitorTest.scala
@@ -17,12 +17,12 @@ class MonitorTest extends FunSuite with SharedSparkContext {
 
   test("Basic monitoring testing") {
 
-    // Monitor is initialy successful:
-    assert(Monitor.isSuccess())
+    // Monitor is initially successful:
+    assert(Monitor.isSuccess)
     // Here is what a report generated without any additional settings should
     // look like:
     var report = removeTimeStamps(Monitor.logs())
-    assert(report === "[..:..] Begining\n")
+    assert(report === "[..:..] Beginning\n")
 
     // Include additional info which are placed in the report's header:
     Monitor.setTitle("Processing of whatever")
@@ -30,33 +30,30 @@ class MonitorTest extends FunSuite with SharedSparkContext {
     Monitor.addDescription(
       "Documentation: https://github.com/xavierguihot/spark_helper")
     report = removeTimeStamps(Monitor.logs())
-    var expectedReport = (
+    var expectedReport =
       "					Processing of whatever\n" +
         "\n" +
         "Point of contact: x.guihot@gmail.com, smbdy@gmail.com\n" +
         "Documentation: https://github.com/xavierguihot/spark_helper\n" +
-        "[..:..] Begining\n"
-    )
+        "[..:..] Beginning\n"
     assert(report === expectedReport)
 
     // Simple text update without success modification:
     Monitor.reset()
     Monitor.log("My First Stage")
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] My First Stage\n"
-    )
     assert(report === expectedReport)
 
     // Let's call .log() another time:
     Monitor.log("My Second Stage")
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] My First Stage\n" +
         "[..:..-..:..] My Second Stage\n"
-    )
     assert(report === expectedReport)
 
     // Successive updates:
@@ -64,33 +61,30 @@ class MonitorTest extends FunSuite with SharedSparkContext {
     Monitor.reset()
     Monitor.success("My First Stage")
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] My First Stage: success\n"
-    )
     assert(report === expectedReport)
-    assert(Monitor.isSuccess())
+    assert(Monitor.isSuccess)
     // Update report with a failure:
     Monitor.error("My Second Stage")
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] My First Stage: success\n" +
         "[..:..-..:..] My Second Stage: failed\n"
-    )
     assert(report === expectedReport)
-    assert(!Monitor.isSuccess())
+    assert(!Monitor.isSuccess)
     // A success after a failure, which must not overwrite the failure:
     Monitor.success("My Third Stage")
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] My First Stage: success\n" +
         "[..:..-..:..] My Second Stage: failed\n" +
         "[..:..-..:..] My Third Stage: success\n"
-    )
     assert(report === expectedReport)
-    assert(!Monitor.isSuccess())
+    assert(!Monitor.isSuccess)
   }
 
   test("Check current.ongoing live monitoring") {
@@ -112,18 +106,17 @@ class MonitorTest extends FunSuite with SharedSparkContext {
       .toList
       .mkString("\n")
 
-    val expectedReport = (
+    val expectedReport =
       "					My Processing\n" +
         "\n" +
         "Point of contact: x.guihot@gmail.com, smbdy@gmail.com\n" +
         "Documentation: https://github.com/xavierguihot/spark_helper\n" +
-        "[..:..] Begining\n" +
+        "[..:..] Beginning\n" +
         "[..:..-..:..] Doing something\n" +
         "\n" +
         "WARNING: If this file exists it does not necessarily mean that " +
         "your job is still running. This file might persist if your job has " +
         "been killed and thus couldn't reach your call to the Monitor.store()."
-    )
     assert(removeTimeStamps(reportStoredLines) === expectedReport)
   }
 
@@ -132,7 +125,7 @@ class MonitorTest extends FunSuite with SharedSparkContext {
     Monitor.reset()
 
     // Explanation to someone running tests and seeing an error stack trace
-    // even though tests are actually successfull:
+    // even though tests are actually successful:
     println(
       "README: The following stack trace is NOT a test failure. This " +
         "is the logging/print of the tested stack trace error as it would " +
@@ -146,14 +139,13 @@ class MonitorTest extends FunSuite with SharedSparkContext {
         Monitor.error(nfe, "Parse to integer", "my diagnostic")
     }
     // Warning, here I remove the stack trace because it depends on the
-    // java/scala version! And yes this test is a bit less usefull.
+    // java/scala version! And yes this test is a bit less useful.
     val report =
       removeTimeStamps(Monitor.logs()).split("\n").take(3).mkString("\n")
-    val expectedReport = (
-      "[..:..] Begining\n" +
+    val expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] Parse to integer: failed\n" +
         "	Diagnostic: my diagnostic"
-    )
     assert(report === expectedReport)
   }
 
@@ -171,11 +163,11 @@ class MonitorTest extends FunSuite with SharedSparkContext {
     )
 
     assert(!success)
-    assert(!Monitor.isSuccess())
+    assert(!Monitor.isSuccess)
 
     var report = removeTimeStamps(Monitor.logs())
-    var expectedReport = (
-      "[..:..] Begining\n" +
+    var expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] Tests for whatever: failed\n" +
         "	KPI: pctOfWhatever\n" +
         "		Value: 0.06%\n" +
@@ -189,7 +181,6 @@ class MonitorTest extends FunSuite with SharedSparkContext {
         "		Value: 1235.0\n" +
         "		Must be equal to 1235.0\n" +
         "		Validated: true\n"
-    )
     assert(report === expectedReport)
 
     // 2: Single test:
@@ -199,17 +190,16 @@ class MonitorTest extends FunSuite with SharedSparkContext {
       "Tests for whatever")
 
     assert(success)
-    assert(Monitor.isSuccess())
+    assert(Monitor.isSuccess)
 
     report = removeTimeStamps(Monitor.logs())
-    expectedReport = (
-      "[..:..] Begining\n" +
+    expectedReport =
+      "[..:..] Beginning\n" +
         "[..:..-..:..] Tests for whatever: success\n" +
         "	KPI: someNbr\n" +
         "		Value: 5.5E7\n" +
         "		Must be superior than 5.0E7\n" +
         "		Validated: true\n"
-    )
     assert(report === expectedReport)
   }
 
@@ -235,15 +225,14 @@ class MonitorTest extends FunSuite with SharedSparkContext {
       .mkString("\n")
       .dropRight(2) + "00" // removes the seconds of the job duration
 
-    val expectedReport = (
+    val expectedReport =
       "					My Processing\n" +
         "\n" +
         "Point of contact: x.guihot@gmail.com\n" +
         "Documentation: https://github.com/xavierguihot/spark_helper\n" +
-        "[..:..] Begining\n" +
+        "[..:..] Beginning\n" +
         "[..:..-..:..] Doing something: success\n" +
         "[..:..] Duration: 00:00:00"
-    )
     assert(removeTimeStamps(reportStoredLines) === expectedReport)
   }
 
@@ -296,7 +285,7 @@ class MonitorTest extends FunSuite with SharedSparkContext {
           timeStampFreeLogs.substring(0, index) + "[..:..-..:..]" +
             timeStampFreeLogs.substring(index + 13)
 
-      index = timeStampFreeLogs.indexOf("[", index + 1);
+      index = timeStampFreeLogs.indexOf("[", index + 1)
     }
 
     timeStampFreeLogs
diff --git a/src/test/scala/com/spark_helper/SparkHelperTest.scala b/src/test/scala/com/spark_helper/SparkHelperTest.scala
index 21d01e0..70d706d 100644
--- a/src/test/scala/com/spark_helper/SparkHelperTest.scala
+++ b/src/test/scala/com/spark_helper/SparkHelperTest.scala
@@ -75,35 +75,30 @@ class SparkHelperTest
 
     val weirdFormatFilePath = s"$resourceFolder/some_weird_format.txt"
 
-    // 1: Let's read a file where a record begins with a line begining with
-    // 3 and other lines begining by 4:
+    // 1: Let's read a file where a record begins with a line beginning with
+    // 3 and other lines beginning by 4:
 
     HdfsHelper.deleteFile(weirdFormatFilePath)
 
-    val textContent = (
+    val textContent =
       "3 first line of the first record\n" +
         "4 another line of the first record\n" +
         "4 and another one for the first record\n" +
         "3 first line of the second record\n" +
         "3 first line of the third record\n" +
         "4 another line for the third record"
-    )
 
     HdfsHelper.writeToHdfsFile(textContent, weirdFormatFilePath)
 
     var computedRecords = sc.textFile(weirdFormatFilePath, "\n3").collect()
 
     var expectedRecords = Array(
-      (
-        "3 first line of the first record\n" +
-          "4 another line of the first record\n" +
-          "4 and another one for the first record"
-      ),
+      "3 first line of the first record\n" +
+        "4 another line of the first record\n" +
+        "4 and another one for the first record",
       " first line of the second record",
-      (
-        " first line of the third record\n" +
-          "4 another line for the third record"
-      )
+      " first line of the third record\n" +
+        "4 another line for the third record"
     )
 
     assert(computedRecords === expectedRecords)
@@ -132,15 +127,11 @@ class SparkHelperTest
 
     expectedRecords = Array(
       "<Customers>\n",
-      (
-        "<Address>34 thingy street, someplace, sometown</Address>\n" +
-          "</Customer>\n"
-      ),
-      (
-        "<Address>12 thingy street, someplace, sometown</Address>\n" +
-          "</Customer>\n" +
-          "</Customers>"
-      )
+      "<Address>34 thingy street, someplace, sometown</Address>\n" +
+        "</Customer>\n",
+      "<Address>12 thingy street, someplace, sometown</Address>\n" +
+        "</Customer>\n" +
+        "</Customers>"
     )
 
     assert(computedRecords === expectedRecords)
@@ -167,7 +158,7 @@ class SparkHelperTest
 
     val keyValueFolder = s"$resourceFolder/key_value_storage"
 
-    // 1: Let's strore key values per file:
+    // 1: Let's store key values per file:
 
     HdfsHelper.deleteFolder(keyValueFolder)
 
@@ -189,9 +180,9 @@ class SparkHelperTest
     assert(HdfsHelper.folderExists(keyValueFolder))
 
     // And it contains one file per key:
-    var genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    var generatedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
     var expectedKeyFiles = List("_SUCCESS", "key_1", "key_2", "key_3")
-    assert(genratedKeyFiles === expectedKeyFiles)
+    assert(generatedKeyFiles === expectedKeyFiles)
 
     var valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1").collect().sorted
     assert(valuesForKey1 === Array("value_a", "value_b"))
@@ -202,7 +193,7 @@ class SparkHelperTest
     val valuesForKey3 = sc.textFile(s"$keyValueFolder/key_3").collect().sorted
     assert(valuesForKey3 === Array("value_a", "value_b"))
 
-    // 2: Let's strore key values per file; but without providing the nbr of
+    // 2: Let's store key values per file; but without providing the nbr of
     // keys:
 
     HdfsHelper.deleteFolder(keyValueFolder)
@@ -213,14 +204,14 @@ class SparkHelperTest
     assert(HdfsHelper.folderExists(keyValueFolder))
 
     // And it contains one file per key:
-    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    generatedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
     expectedKeyFiles = List("_SUCCESS", "key_1", "key_2", "key_3")
-    assert(genratedKeyFiles === expectedKeyFiles)
+    assert(generatedKeyFiles === expectedKeyFiles)
 
     valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1").collect().sorted
     assert(valuesForKey1 === Array("value_a", "value_b"))
 
-    // 3: Let's strore key values per file and compress these files:
+    // 3: Let's store key values per file and compress these files:
 
     HdfsHelper.deleteFolder(keyValueFolder)
 
@@ -230,9 +221,9 @@ class SparkHelperTest
     assert(HdfsHelper.folderExists(keyValueFolder))
 
     // And it contains one file per key:
-    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
+    generatedKeyFiles = HdfsHelper.listFileNamesInFolder(keyValueFolder)
     expectedKeyFiles = List("_SUCCESS", "key_1.gz", "key_2.gz", "key_3.gz")
-    assert(genratedKeyFiles === expectedKeyFiles)
+    assert(generatedKeyFiles === expectedKeyFiles)
 
     valuesForKey1 = sc.textFile(s"$keyValueFolder/key_1.gz").collect().sorted
     assert(valuesForKey1 === Array("value_a", "value_b"))
@@ -254,9 +245,9 @@ class SparkHelperTest
     rddToStore.saveAsTextFileAndCoalesce(testFolder, 2)
 
     // Let's check the nbr of partitions:
-    var genratedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
+    var generatedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
     var expectedKeyFiles = List("_SUCCESS", "part-00000", "part-00001")
-    assert(genratedKeyFiles === expectedKeyFiles)
+    assert(generatedKeyFiles === expectedKeyFiles)
 
     // And let's check the content:
     var singleFileStoredData = sc.textFile(testFolder).collect().sorted
@@ -269,9 +260,9 @@ class SparkHelperTest
     rddToStore.saveAsTextFileAndCoalesce(testFolder, 2, classOf[GzipCodec])
 
     // Let's check the nbr of partitions:
-    genratedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
+    generatedKeyFiles = HdfsHelper.listFileNamesInFolder(testFolder)
     expectedKeyFiles = List("_SUCCESS", "part-00000.gz", "part-00001.gz")
-    assert(genratedKeyFiles === expectedKeyFiles)
+    assert(generatedKeyFiles === expectedKeyFiles)
 
     // And let's check the content:
     singleFileStoredData = sc.textFile(testFolder).collect().sorted
@@ -350,7 +341,7 @@ class SparkHelperTest
       .map {
         case (filePath, line) =>
           val nonLocalPath = filePath.split("src/test/") match {
-            case Array(localPartOfPath, projectRelativePath) =>
+            case Array(_, projectRelativePath) =>
               "file:/.../src/test/" + projectRelativePath
           }
           (nonLocalPath, line)
@@ -411,7 +402,7 @@ class SparkHelperTest
 
     val in = sc.parallelize(Array(1, 3, 2, 7, 8))
     val computedOut = in.partialMap { case a if a % 2 == 0 => 2 * a }
-    val expetcedOut = sc.parallelize(Array(1, 3, 4, 7, 16))
-    assertRDDEquals(computedOut, expetcedOut)
+    val expectedOut = sc.parallelize(Array(1, 3, 4, 7, 16))
+    assertRDDEquals(computedOut, expectedOut)
   }
 }

From 0440f99a9cb9b2454cdca2b69cdf15fa8205adda Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sun, 17 Jun 2018 13:14:22 +0100
Subject: [PATCH 24/25] Update doc

---
 README.md                                     |  27 +-
 .../DateHelper$$IntExtensions.html            | 599 ++++++++++++++++++
 .../DateHelper$$StringExtensions.html         | 595 +++++++++++++++++
 docs/com/spark_helper/DateHelper$.html        | 335 ++++++++--
 .../HdfsHelper$$SeqExtensions.html            | 539 ++++++++++++++++
 .../HdfsHelper$$StringExtensions.html         | 536 ++++++++++++++++
 docs/com/spark_helper/HdfsHelper$.html        | 152 ++++-
 docs/com/spark_helper/Monitor$.html           |  56 +-
 .../SparkHelper$$OptionRDDExtensions.html     | 540 ++++++++++++++++
 .../SparkHelper$$PairRDDExtensions.html       | 376 +++++++++++
 .../SparkHelper$$RDDExtensions.html           | 547 ++++++++++++++++
 .../SparkHelper$$SeqRDDExtensions.html        | 540 ++++++++++++++++
 .../SparkHelper$$SparkContextExtensions.html  | 476 ++++++++++++++
 .../SparkHelper$$StringRDDExtensions.html     | 422 ++++++++++++
 docs/com/spark_helper/SparkHelper$.html       | 481 ++++----------
 docs/com/spark_helper/monitoring/Test.html    |   4 +-
 .../monitoring/ThresholdType.html             |   2 +-
 docs/com/spark_helper/monitoring/package.html |   4 +-
 docs/com/spark_helper/package.html            | 150 +++--
 docs/index.html                               |  14 +-
 docs/index.js                                 |   2 +-
 docs/index/index-a.html                       |   3 +
 docs/index/index-d.html                       |  11 +-
 docs/index/index-f.html                       |   3 +
 docs/index/index-i.html                       |   9 +
 docs/index/index-n.html                       |   5 +-
 docs/index/index-o.html                       |  20 +
 docs/index/index-p.html                       |   8 +-
 docs/index/index-r.html                       |   6 +
 docs/index/index-s.html                       |  60 +-
 docs/index/index-t.html                       |  12 +-
 docs/index/index-w.html                       |   3 +
 docs/org/apache/package.html                  | 118 ++++
 docs/org/apache/spark/TextFileOverwrite$.html | 499 +++++++++++++++
 docs/org/apache/spark/package.html            | 246 +++++++
 docs/org/package.html                         | 118 ++++
 docs/package.html                             |  17 +
 .../scala/com/spark_helper/DateHelper.scala   |  33 +-
 .../scala/com/spark_helper/HdfsHelper.scala   |  64 +-
 src/main/scala/com/spark_helper/Monitor.scala |   3 +-
 .../scala/com/spark_helper/SparkHelper.scala  |  40 +-
 41 files changed, 7077 insertions(+), 598 deletions(-)
 create mode 100644 docs/com/spark_helper/DateHelper$$IntExtensions.html
 create mode 100644 docs/com/spark_helper/DateHelper$$StringExtensions.html
 create mode 100644 docs/com/spark_helper/HdfsHelper$$SeqExtensions.html
 create mode 100644 docs/com/spark_helper/HdfsHelper$$StringExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$OptionRDDExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$PairRDDExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$RDDExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$SeqRDDExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$SparkContextExtensions.html
 create mode 100644 docs/com/spark_helper/SparkHelper$$StringRDDExtensions.html
 create mode 100644 docs/index/index-o.html
 create mode 100644 docs/org/apache/package.html
 create mode 100644 docs/org/apache/spark/TextFileOverwrite$.html
 create mode 100644 docs/org/apache/spark/package.html
 create mode 100644 docs/org/package.html

diff --git a/README.md b/README.md
index 26818eb..50eb2bd 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ The full list of methods is available at
 Contains basic file-related methods mostly based on hdfs apache Hadoop
 FileSystem API [org.apache.hadoop.fs.FileSystem](https://hadoop.apache.org/docs/r2.6.1/api/org/apache/hadoop/fs/FileSystem.html).
 
-A non-exhaustive list of exemples:
+A non-exhaustive list of examples:
 
 ```scala
 import com.spark_helper.HdfsHelper
@@ -58,7 +58,7 @@ HdfsHelper.appendHeader("my/hdfs/file/path.csv", "colum0,column1")
 HdfsHelper.isHdfsXmlCompliantWithXsd("my/hdfs/file/path.xml", getClass.getResource("/some_xml.xsd"))
 HdfsHelper.loadXmlFileFromHdfs("my/hdfs/file/path.xml")
 
-// Very handy to load a config (typesafe format) stored on hdfs at the begining of a spark job:
+// Very handy to load a config (typesafe format) stored on hdfs at the beginning of a spark job:
 HdfsHelper.loadTypesafeConfigFromHdfs("my/hdfs/file/path.conf"): Config
 
 // In order to write small amount of data in a file on hdfs without the whole spark stack:
@@ -89,13 +89,13 @@ The full list of methods is available at
 
 Contains basic RRD-related methods.
 
-A non-exhaustive list of exemples:
+A non-exhaustive list of examples:
 
 ```scala
 import com.spark_helper.SparkHelper._
 
 // Same as rdd.saveAsTextFile("path"), but the result is a single file (while
-// keeping the processing parallelized):
+// keeping the processing distributed):
 rdd.saveAsSingleTextFile("/my/output/file/path.txt")
 rdd.saveAsSingleTextFile("/my/output/file/path.txt", classOf[BZip2Codec])
 
@@ -118,7 +118,7 @@ sc.textFileWithFileName("/my/input/folder/path")
 // RDD. Within each file (named from the key) are all values for this key:
 rdd.saveAsTextFileByKey("/my/output/folder/path")
 
-// Concept mapper (the following exemple transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
+// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
 rdd.partialMap { case a if a % 2 == 0 => 2 * a }
 
 // For when input files contain commas and textFile can't handle it:
@@ -133,7 +133,7 @@ The full list of methods is available at
 Wrapper around [joda-time](http://www.joda.org/joda-time/apidocs/) for
 data-mining classic dates manipulations and job scheduling.
 
-A non-exhaustive list of exemples:
+A non-exhaustive list of examples:
 
 ```scala
 import com.spark_helper.DateHelper
@@ -246,7 +246,7 @@ if (Monitor.isSuccess) {
 // HDFS (this saves the logs in the folder set with Monitor.setLogFolder):
 Monitor.store()
 
-// At the end of the job, if the job isn't successfull, you might want to
+// At the end of the job, if the job isn't successful, you might want to
 // crash it (for instance to get a notification from your scheduler):
 if (!Monitor.isSuccess) throw new Exception() // or send an email, or ...
 ```
@@ -261,7 +261,7 @@ Here are some possible reports generated by the previous pipeline:
 
 My job description (whatever you want); for instance:
 Documentation: https://github.com/xavierguihot/spark_helper
-[10:23] Begining
+[10:23] Beginning
 [10:23-10:23] My pipeline descirption: failed
   Diagnostic: No input data!
     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs://my/hdfs/input/path
@@ -278,7 +278,7 @@ or
 
 My job description (whatever you want); for instance:
 Documentation: https://github.com/xavierguihot/spark_helper
-[10:23] Begining
+[10:23] Beginning
 [10:23-10:41] My pipeline descirption: success
   KPI: Nbr of output records
     Value: 14669071.0
@@ -295,7 +295,7 @@ Documentation: https://github.com/xavierguihot/spark_helper
 ## Including spark_helper to your dependencies:
 
 
-With sbt, add these lines to your build.sbt:
+With sbt:
 
 ```scala
 resolvers += "jitpack" at "https://jitpack.io"
@@ -303,7 +303,7 @@ resolvers += "jitpack" at "https://jitpack.io"
 libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.1.1"
 ```
 
-With maven, add these lines to your pom.xml:
+With maven:
 
 ```xml
 <repositories>
@@ -320,7 +320,7 @@ With maven, add these lines to your pom.xml:
 </dependency>
 ```
 
-With gradle, add these lines to your build.gradle:
+With gradle:
 
 ```groovy
 allprojects {
@@ -333,3 +333,6 @@ dependencies {
 	compile 'com.github.xavierguihot:spark_helper:v1.1.1'
 }
 ```
+
+For versions anterior to `2.0.0`, use prefix `v` in the version tag; for
+instance `v1.0.0`
diff --git a/docs/com/spark_helper/DateHelper$$IntExtensions.html b/docs/com/spark_helper/DateHelper$$IntExtensions.html
new file mode 100644
index 0000000..9b5203c
--- /dev/null
+++ b/docs/com/spark_helper/DateHelper$$IntExtensions.html
@@ -0,0 +1,599 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>IntExtensions - com.spark_helper.DateHelper.IntExtensions</title>
+          <meta name="description" content="IntExtensions - com.spark helper.DateHelper.IntExtensions" />
+          <meta name="keywords" content="IntExtensions com.spark helper.DateHelper.IntExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.DateHelper$$IntExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a></p>
+        <h1>IntExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">package DateHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">IntExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.DateHelper.IntExtensions"><span>IntExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.DateHelper.IntExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(int:Int):com.spark_helper.DateHelper.IntExtensions"></a>
+      <a id="&lt;init&gt;:IntExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">IntExtensions</span><span class="params">(<span name="int">int: <span class="extype" name="scala.Int">Int</span></span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@&lt;init&gt;(int:Int):com.spark_helper.DateHelper.IntExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.IntExtensions#daysAfter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="daysAfter(date:String):String"></a>
+      <a id="daysAfter(String):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">daysAfter</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@daysAfter(date:String):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it will be x days after the given date under the
+default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after the given date under the
+default format.</p><p>If the given date is &quot;20170122&quot; and we request the date it will be 3
+days after, we'll return &quot;20170125&quot;.</p><pre>assert(<span class="num">5.</span>daysAfter(<span class="lit">"20170305"</span>) == <span class="lit">"20170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date under the default format for which we want the date
+for nbrOfDaysAfter days after.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysAfter after date under the default
+format.</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.IntExtensions#daysAfter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="daysAfter:String"></a>
+      <a id="daysAfter:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">daysAfter</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@daysAfter:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it will be x days after today under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after today under the default format.</p><p>If we're &quot;20170125&quot; and we request for 3 days after, we'll return
+&quot;20170127&quot;.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(<span class="num">3.</span>daysAfter == <span class="lit">"20170313"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>today's date plus the given nbr of days</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.IntExtensions#daysAgo" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="daysAgo:String"></a>
+      <a id="daysAgo:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">daysAgo</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@daysAgo:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it was x days before today under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before today under the default format.</p><p>If we're &quot;20170125&quot; and we request for 3 days before, we'll return
+&quot;20170122&quot;.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(<span class="num">3.</span>daysAgo == <span class="lit">"20170307"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>today's date minus the given nbr of days</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.IntExtensions#daysBefore" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="daysBefore(date:String):String"></a>
+      <a id="daysBefore(String):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">daysBefore</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@daysBefore(date:String):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it was x days before the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before the given date.</p><p>If the given date is &quot;20170125&quot; and we request the date it was 3 days
+before, this will return &quot;20170122&quot;.</p><pre>assert(<span class="num">3.</span>daysBefore(<span class="lit">"20170310"</span>) == <span class="lit">"20170307"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date under the default format for which we want the date
+for nbrOfDaysBefore days before.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysBefore before date under the default
+format.</p></dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.IntExtensions#int" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="int:Int"></a>
+      <a id="int:Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">int</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@int:Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$IntExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/DateHelper$$StringExtensions.html b/docs/com/spark_helper/DateHelper$$StringExtensions.html
new file mode 100644
index 0000000..cf0c71c
--- /dev/null
+++ b/docs/com/spark_helper/DateHelper$$StringExtensions.html
@@ -0,0 +1,595 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>StringExtensions - com.spark_helper.DateHelper.StringExtensions</title>
+          <meta name="description" content="StringExtensions - com.spark helper.DateHelper.StringExtensions" />
+          <meta name="keywords" content="StringExtensions com.spark helper.DateHelper.StringExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.DateHelper$$StringExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a></p>
+        <h1>StringExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">package DateHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.DateHelper.StringExtensions"><span>StringExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.DateHelper.StringExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(string:String):com.spark_helper.DateHelper.StringExtensions"></a>
+      <a id="&lt;init&gt;:StringExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringExtensions</span><span class="params">(<span name="string">string: <span class="extype" name="scala.Predef.String">String</span></span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@&lt;init&gt;(string:String):com.spark_helper.DateHelper.StringExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.StringExtensions#isCompliantWith" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isCompliantWith(format:String):Boolean"></a>
+      <a id="isCompliantWith(String):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isCompliantWith</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@isCompliantWith(format:String):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Validates the formatted date is compliant with the provided format.</p><div class="fullcomment"><div class="comment cmt"><p>Validates the formatted date is compliant with the provided format.</p><pre>assert(<span class="lit">"20170302"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">"20170333"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(<span class="lit">"20170228"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">"20170229"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">"170228"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">""</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">"a"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))
+assert(!<span class="lit">"24JAN17"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>))</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>if the provided date is under the provided format</p></dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.StringExtensions#nextDay" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nextDay:String"></a>
+      <a id="nextDay:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nextDay</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@nextDay:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns the date one day after the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the date one day after the given date.</p><pre>assert(<span class="lit">"20170310"</span>.nextDay == <span class="lit">"20170311"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>the date of the day after the given date</p></dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.StringExtensions#previousDay" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="previousDay:String"></a>
+      <a id="previousDay:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">previousDay</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@previousDay:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns the date one day before the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the date one day before the given date.</p><pre>assert(<span class="lit">"20170310"</span>.previousDay == <span class="lit">"20170309"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>the date of the day before the given date</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.StringExtensions#string" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="string:String"></a>
+      <a id="string:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">string</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@string:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper.StringExtensions#to" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="to(lastDate:String):List[String]"></a>
+      <a id="to(String):List[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">to</span><span class="params">(<span name="lastDate">lastDate: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.List">List</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@to(lastDate:String):List[String]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Creates the list of dates between the two given dates.</p><div class="fullcomment"><div class="comment cmt"><p>Creates the list of dates between the two given dates.</p><pre>assert((<span class="lit">"20161230"</span> to <span class="lit">"20170101"</span>) == <span class="std">List</span>(<span class="lit">"20161230"</span>, <span class="lit">"20161231"</span>, <span class="lit">"20170101"</span>))</pre></div><dl class="paramcmts block"><dt class="param">lastDate</dt><dd class="cmt"><p>the last date</p></dd><dt>returns</dt><dd class="cmt"><p>the list of dates between this string and the lastDate in the
+default format.</p></dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$$StringExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/DateHelper$.html b/docs/com/spark_helper/DateHelper$.html
index 77a52dd..749074c 100644
--- a/docs/com/spark_helper/DateHelper$.html
+++ b/docs/com/spark_helper/DateHelper$.html
@@ -52,14 +52,31 @@ <h4 id="signature" class="signature">
           <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>A facility which deals with usual date needs (wrapper around
 <a href="http://www.joda.org/joda-time/apidocs/">joda-time</a>).</p><p>The goal is to remove the maximum of highly used low-level code from your
 spark job and replace it with methods fully tested whose name is
-self-explanatory/readable.</p><p>A few exemples:</p><pre>assert(DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) == <span class="std">List</span>(<span class="lit">"20161230"</span>, <span class="lit">"20161231"</span>, <span class="lit">"20170101"</span>))
-assert(DateHelper.today() == <span class="lit">"20170310"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.yesterday() == <span class="lit">"20170309"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.reformatDate(<span class="lit">"20170327"</span>, <span class="lit">"yyyyMMdd"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170327"</span>)
-assert(DateHelper.now(<span class="lit">"HH:mm"</span>) == <span class="lit">"10:24"</span>)
-assert(DateHelper.currentTimestamp() == <span class="lit">"1493105229736"</span>)
-assert(DateHelper.nDaysBefore(<span class="num">3</span>) == <span class="lit">"20170307"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.nDaysAfterDate(<span class="num">3</span>, <span class="lit">"20170307"</span>) == <span class="lit">"20170310"</span>)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
+self-explanatory/readable.</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.DateHelper
+
+DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) <span class="cmt">// List("20161230", "20161231", "20170101")</span>
+DateHelper.today <span class="cmt">// "20170310"</span>
+DateHelper.yesterday <span class="cmt">// "20170309"</span>
+DateHelper.reformatDate(<span class="lit">"20170327"</span>, <span class="lit">"yyyyMMdd"</span>, <span class="lit">"yyMMdd"</span>) <span class="cmt">// "170327"</span>
+DateHelper.now(<span class="lit">"HH:mm"</span>) <span class="cmt">// "10:24"</span>
+DateHelper.currentTimestamp <span class="cmt">// "1493105229736"</span>
+DateHelper.nDaysBefore(<span class="num">3</span>) <span class="cmt">// "20170307"</span>
+DateHelper.nDaysAfterDate(<span class="num">3</span>, <span class="lit">"20170307"</span>) <span class="cmt">// "20170310"</span>
+DateHelper.nextDay(<span class="lit">"20170310"</span>) <span class="cmt">// "20170311"</span>
+DateHelper.nbrOfDaysSince(<span class="lit">"20170302"</span>) <span class="cmt">// 8</span>
+DateHelper.nbrOfDaysBetween(<span class="lit">"20170327"</span>, <span class="lit">"20170401"</span>) <span class="cmt">// 5</span>
+DateHelper.dayOfWeek(<span class="lit">"20160614"</span>) <span class="cmt">// 2</span>
+
+<span class="kw">import</span> com.spark_helper.DateHelper._
+
+<span class="num">2.</span>daysAgo <span class="cmt">// "20170308"</span>
+<span class="lit">"20161230"</span> to <span class="lit">"20170101"</span> <span class="cmt">// List("20161230", "20161231", "20170101")</span>
+<span class="num">3.</span>daysBefore(<span class="lit">"20170310"</span>) <span class="cmt">// "20170307"</span>
+<span class="num">5.</span>daysAfter <span class="cmt">// "20170315"</span>
+<span class="num">4.</span>daysAfter(<span class="lit">"20170310"</span>) <span class="cmt">// "20170314"</span>
+<span class="lit">"20170302"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>)
+<span class="lit">"20170310"</span>.nextDay <span class="cmt">// "20170311"</span>
+<span class="lit">"20170310"</span>.previousDay <span class="cmt">// "20170309"</span></pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/DateHelper.scala">DateHelper</a>
 </p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl><div class="toggleContainer block">
           <span class="toggle">Linear Supertypes</span>
@@ -100,7 +117,44 @@ <h4 id="signature" class="signature">
         <div id="allMembers">
         
 
-        
+        <div id="types" class="types members">
+              <h3>Type Members</h3>
+              <ol><li name="com.spark_helper.DateHelper.IntExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="IntExtensionsextendsAnyRef"></a>
+      <a id="IntExtensions:IntExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="DateHelper$$IntExtensions.html"><span class="name">IntExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@IntExtensionsextendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.DateHelper.StringExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="StringExtensionsextendsAnyRef"></a>
+      <a id="StringExtensions:StringExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="DateHelper$$StringExtensions.html"><span class="name">StringExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@StringExtensionsextendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
 
         
 
@@ -198,18 +252,18 @@ <h4 class="signature">
               
         </dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#currentTimestamp" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="currentTimestamp():String"></a>
-      <a id="currentTimestamp():String"></a>
+      <a id="currentTimestamp:String"></a>
+      <a id="currentTimestamp:String"></a>
       <h4 class="signature">
       <span class="modifier_kind">
         <span class="modifier"></span>
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">currentTimestamp</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">currentTimestamp</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.DateHelper$@currentTimestamp():String" title="Permalink" target="_top">
+      <a href="../../index.html#com.spark_helper.DateHelper$@currentTimestamp:String" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
@@ -241,7 +295,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">dateFromTimestamp</span><span class="params">(<span name="timestamp">timestamp: <span class="extype" name="scala.Long">Long</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">dateFromTimestamp</span><span class="params">(<span name="timestamp">timestamp: <span class="extype" name="scala.Long">Long</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@dateFromTimestamp(timestamp:Long,format:String):String" title="Permalink" target="_top">
@@ -250,7 +304,7 @@ <h4 class="signature">
     </span>
       <p class="shortcomment cmt">Returns the date associated to the given UTC timestamp.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the date associated to the given UTC timestamp.</p><pre>assert(DateHelper.dateFromTimestamp(<span class="num">1496074819</span>L) == <span class="lit">"20170529"</span>)
 assert(DateHelper.dateFromTimestamp(<span class="num">1496074819</span>L, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170529"</span>)</pre></div><dl class="paramcmts block"><dt class="param">timestamp</dt><dd class="cmt"><p>the UTC timestamps (nbr of millis since 1970-01-01) for
-which to get the associated date.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format of the provided dates</p></dd><dt>returns</dt><dd class="cmt"><p>the associated date under the requested format</p></dd></dl></div>
+which to get the associated date.</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format of the provided dates</p></dd><dt>returns</dt><dd class="cmt"><p>the associated date under the requested format</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#dayOfWeek" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="dayOfWeek(date:String,format:String):Int"></a>
       <a id="dayOfWeek(String,String):Int"></a>
@@ -260,15 +314,14 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">dayOfWeek</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+        <span class="name">dayOfWeek</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@dayOfWeek(date:String,format:String):Int" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns the day of week for a date under the given format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the day of week for a date under the given format.</p><p>A Monday is 1 and a Sunday is 7.</p><pre>assert(DateHelper.dayOfWeek(<span class="lit">"20160614"</span>) == <span class="num">2</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to get the day of week</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format under which the date is
-provided.</p></dd><dt>returns</dt><dd class="cmt"><p>the associated day of week, such as 2 for Tuesday</p></dd></dl></div>
+      <p class="shortcomment cmt">Returns the day of week for a date under the given format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the day of week for a date under the given format.</p><p>A Monday is 1 and a Sunday is 7.</p><pre>assert(DateHelper.dayOfWeek(<span class="lit">"20160614"</span>) == <span class="num">2</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to get the day of week</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format under which the date is provided</p></dd><dt>returns</dt><dd class="cmt"><p>the associated day of week, such as 2 for Tuesday</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#daysBetween" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="daysBetween(firstDate:String,lastDate:String,format:String):List[String]"></a>
       <a id="daysBetween(String,String,String):List[String]"></a>
@@ -278,15 +331,15 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">daysBetween</span><span class="params">(<span name="firstDate">firstDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lastDate">lastDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.List">List</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+        <span class="name">daysBetween</span><span class="params">(<span name="firstDate">firstDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lastDate">lastDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.List">List</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@daysBetween(firstDate:String,lastDate:String,format:String):List[String]" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Finds the list of dates between the two given dates.</p><div class="fullcomment"><div class="comment cmt"><p>Finds the list of dates between the two given dates.</p><pre>assert(DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) == <span class="std">List</span>(<span class="lit">"20161230"</span>, <span class="lit">"20161231"</span>, <span class="lit">"20170101"</span>))</pre></div><dl class="paramcmts block"><dt class="param">firstDate</dt><dd class="cmt"><p>the first date (in the given format)</p></dd><dt class="param">lastDate</dt><dd class="cmt"><p>the last date (in the given format)</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format to use for firstDate and
-lastDate and for the returned list of dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the list of dates between firstDate and lastDate in the given
+      <p class="shortcomment cmt">Finds the list of dates between the two given dates.</p><div class="fullcomment"><div class="comment cmt"><p>Finds the list of dates between the two given dates.</p><pre>assert(DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) == <span class="std">List</span>(<span class="lit">"20161230"</span>, <span class="lit">"20161231"</span>, <span class="lit">"20170101"</span>))</pre></div><dl class="paramcmts block"><dt class="param">firstDate</dt><dd class="cmt"><p>the first date (in the given format)</p></dd><dt class="param">lastDate</dt><dd class="cmt"><p>the last date (in the given format)</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format to use for firstDate and lastDate and for the
+returned list of dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the list of dates between firstDate and lastDate in the given
 format.</p></dd></dl></div>
     </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="eq(x$1:AnyRef):Boolean"></a>
@@ -402,7 +455,7 @@ <h4 class="signature">
 assert(!DateHelper.isDateCompliantWithFormat(<span class="lit">"170228"</span>, <span class="lit">"yyyyMMdd"</span>))
 assert(!DateHelper.isDateCompliantWithFormat(<span class="lit">""</span>, <span class="lit">"yyyyMMdd"</span>))
 assert(!DateHelper.isDateCompliantWithFormat(<span class="lit">"a"</span>, <span class="lit">"yyyyMMdd"</span>))
-assert(!DateHelper.isDateCompliantWithFormat(<span class="lit">"24JAN17"</span>, <span class="lit">"yyyyMMdd"</span>))</pre></div><dl class="paramcmts block"><dt class="param">stringValue</dt><dd class="cmt"><p>the stringified date</p></dd><dt>returns</dt><dd class="cmt"><p>if the provided date is under the provided format</p></dd></dl></div>
+assert(!DateHelper.isDateCompliantWithFormat(<span class="lit">"24JAN17"</span>, <span class="lit">"yyyyMMdd"</span>))</pre></div><dl class="paramcmts block"><dt class="param">stringValue</dt><dd class="cmt"><p>the formatted date</p></dd><dt>returns</dt><dd class="cmt"><p>if the provided date is under the provided format</p></dd></dl></div>
     </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="isInstanceOf[T0]:Boolean"></a>
       <a id="isInstanceOf[T0]:Boolean"></a>
@@ -440,6 +493,64 @@ <h4 class="signature">
 returned list of dates and thus prefer getting a list of Joda DateTime
 objects instead of String dates.
 </p></div><dl class="paramcmts block"><dt class="param">jodaFirstDate</dt><dd class="cmt"><p>the joda DateTime first date</p></dd><dt class="param">jodaLastDate</dt><dd class="cmt"><p>the joda DateTime last date</p></dd><dt>returns</dt><dd class="cmt"><p>the list of joda DateTime between jodaFirstDate and jodaLastDate</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#nDaysAfter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nDaysAfter(nbrOfDaysAfter:Int):String"></a>
+      <a id="nDaysAfter(Int):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nDaysAfter</span><span class="params">(<span name="nbrOfDaysAfter">nbrOfDaysAfter: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@nDaysAfter(nbrOfDaysAfter:Int):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it will be x days after today under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after today under the default format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.nDaysAfter(<span class="num">5</span>) == <span class="lit">"20170315"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysAfter</dt><dd class="cmt"><p>the nbr of days after today</p></dd><dt>returns</dt><dd class="cmt"><p>today's date plus the nbrOfDaysAfter under the default format</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#nDaysAfter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nDaysAfter(nbrOfDaysAfter:Int,format:String):String"></a>
+      <a id="nDaysAfter(Int,String):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nDaysAfter</span><span class="params">(<span name="nbrOfDaysAfter">nbrOfDaysAfter: <span class="extype" name="scala.Int">Int</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@nDaysAfter(nbrOfDaysAfter:Int,format:String):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it will be x days after today under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after today under the requested format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.nDaysAfter(<span class="num">5</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170315"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysAfter</dt><dd class="cmt"><p>the nbr of days after today</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format for the returned date</p></dd><dt>returns</dt><dd class="cmt"><p>today's date plus the nbrOfDaysAfter under the requested format</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#nDaysAfterDate" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nDaysAfterDate(nbrOfDaysAfter:Int,date:String):String"></a>
+      <a id="nDaysAfterDate(Int,String):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nDaysAfterDate</span><span class="params">(<span name="nbrOfDaysAfter">nbrOfDaysAfter: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@nDaysAfterDate(nbrOfDaysAfter:Int,date:String):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it will be x days after the given date under the
+default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after the given date under the
+default format.</p><p>If the given date is &quot;20170122&quot; and we request the date it will be 3 days
+after, we'll return &quot;20170125&quot;.</p><pre>assert(DateHelper.nDaysAfterDate(<span class="num">5</span>, <span class="lit">"20170305"</span>) == <span class="lit">"20170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysAfter</dt><dd class="cmt"><p>the nbr of days after the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the default format for which we want the date
+for nbrOfDaysAfter days after.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysAfter after date under the default
+format.</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#nDaysAfterDate" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="nDaysAfterDate(nbrOfDaysAfter:Int,date:String,format:String):String"></a>
       <a id="nDaysAfterDate(Int,String,String):String"></a>
@@ -449,7 +560,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nDaysAfterDate</span><span class="params">(<span name="nbrOfDaysAfter">nbrOfDaysAfter: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">nDaysAfterDate</span><span class="params">(<span name="nbrOfDaysAfter">nbrOfDaysAfter: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nDaysAfterDate(nbrOfDaysAfter:Int,date:String,format:String):String" title="Permalink" target="_top">
@@ -457,11 +568,27 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Returns which date it will be x days after the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it will be x days after the given date.</p><p>If the given date is &quot;20170122&quot; and we request the date it will be 3 days
-after, we'll return &quot;20170125&quot;.</p><pre>assert(DateHelper.nDaysAfterDate(<span class="num">3</span>, <span class="lit">"20170307"</span>) == <span class="lit">"20170310"</span>)
-assert(DateHelper.nDaysAfterDate(<span class="num">5</span>, <span class="lit">"170305"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysAfter</dt><dd class="cmt"><p>the nbr of days after the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the provided format for which we want the date
-for nbrOfDaysAfter days after.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format for the provided and
-returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysAfter after date under the requested
+after, we'll return &quot;20170125&quot;.</p><pre>assert(DateHelper.nDaysAfterDate(<span class="num">5</span>, <span class="lit">"170305"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysAfter</dt><dd class="cmt"><p>the nbr of days after the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the provided format for which we want the date
+for nbrOfDaysAfter days after.</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format for the provided and returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysAfter after date under the requested
 format.</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#nDaysBefore" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nDaysBefore(nbrOfDaysBefore:Int):String"></a>
+      <a id="nDaysBefore(Int):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nDaysBefore</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@nDaysBefore(nbrOfDaysBefore:Int):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it was x days before today.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before today.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.nDaysBefore(<span class="num">5</span>) == <span class="lit">"20170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before today</p></dd><dt>returns</dt><dd class="cmt"><p>today's date minus the nbrOfDaysBefore under the default format</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#nDaysBefore" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="nDaysBefore(nbrOfDaysBefore:Int,format:String):String"></a>
       <a id="nDaysBefore(Int,String):String"></a>
@@ -471,17 +598,35 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nDaysBefore</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">nDaysBefore</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nDaysBefore(nbrOfDaysBefore:Int,format:String):String" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns which date it was x days before today under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before today under the requested format.</p><p>If we're &quot;20170125&quot; and we request for 3 days before, we'll return
-&quot;20170122&quot;.</p><pre><span class="cmt">// If today's "20170310":</span>
-assert(DateHelper.nDaysBefore(<span class="num">3</span>) == <span class="lit">"20170307"</span>)
-assert(DateHelper.nDaysBefore(<span class="num">5</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before today</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format for the returned date</p></dd><dt>returns</dt><dd class="cmt"><p>today's date minus the nbrOfDaysBefore under the requested format</p></dd></dl></div>
+      <p class="shortcomment cmt">Returns which date it was x days before today under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before today under the requested format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.nDaysBefore(<span class="num">5</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before today</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format for the returned date</p></dd><dt>returns</dt><dd class="cmt"><p>today's date minus the nbrOfDaysBefore under the requested format</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#nDaysBeforeDate" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="nDaysBeforeDate(nbrOfDaysBefore:Int,date:String):String"></a>
+      <a id="nDaysBeforeDate(Int,String):String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">nDaysBeforeDate</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@nDaysBeforeDate(nbrOfDaysBefore:Int,date:String):String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it was x days before the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before the given date.</p><p>If the given date is &quot;20170125&quot; and we request the date it was 3 days
+before, this will return &quot;20170122&quot;.</p><pre>assert(DateHelper.nDaysBeforeDate(<span class="num">5</span>, <span class="lit">"20170310"</span>) == <span class="lit">"20170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the default format for which we want the date
+for nbrOfDaysBefore days before.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysBefore before date under the default
+format.</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#nDaysBeforeDate" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="nDaysBeforeDate(nbrOfDaysBefore:Int,date:String,format:String):String"></a>
       <a id="nDaysBeforeDate(Int,String,String):String"></a>
@@ -491,7 +636,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nDaysBeforeDate</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">nDaysBeforeDate</span><span class="params">(<span name="nbrOfDaysBefore">nbrOfDaysBefore: <span class="extype" name="scala.Int">Int</span></span>, <span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nDaysBeforeDate(nbrOfDaysBefore:Int,date:String,format:String):String" title="Permalink" target="_top">
@@ -499,10 +644,8 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Returns which date it was x days before the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was x days before the given date.</p><p>If the given date is &quot;20170125&quot; and we request the date it was 3 days
-before, we'll return &quot;20170122&quot;.</p><pre>assert(DateHelper.nDaysBeforeDate(<span class="num">3</span>, <span class="lit">"20170310"</span>) == <span class="lit">"20170307"</span>)
-assert(DateHelper.nDaysBeforeDate(<span class="num">5</span>, <span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the provided format for which we want the date
-for nbrOfDaysBefore days before.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format for the provided and
-returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysBefore before date under the requested
+before, this will return &quot;20170122&quot;.</p><pre>assert(DateHelper.nDaysBeforeDate(<span class="num">5</span>, <span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170305"</span>)</pre></div><dl class="paramcmts block"><dt class="param">nbrOfDaysBefore</dt><dd class="cmt"><p>the nbr of days before the given date</p></dd><dt class="param">date</dt><dd class="cmt"><p>the date under the provided format for which we want the date
+for nbrOfDaysBefore days before.</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format for the provided and returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date it was nbrOfDaysBefore before date under the requested
 format.</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#nbrOfDaysBetween" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="nbrOfDaysBetween(firstDate:String,lastDate:String,format:String):Int"></a>
@@ -513,7 +656,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nbrOfDaysBetween</span><span class="params">(<span name="firstDate">firstDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lastDate">lastDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+        <span class="name">nbrOfDaysBetween</span><span class="params">(<span name="firstDate">firstDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lastDate">lastDate: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nbrOfDaysBetween(firstDate:String,lastDate:String,format:String):Int" title="Permalink" target="_top">
@@ -524,7 +667,7 @@ <h4 class="signature">
 assert(DateHelper.nbrOfDaysBetween(<span class="lit">"20170327"</span>, <span class="lit">"20170401"</span>) == <span class="num">5</span>)</pre><p>This expects the first date to be before the last date.
 </p></div><dl class="paramcmts block"><dt class="param">firstDate</dt><dd class="cmt"><p>the first date of the range for which to egt the nbr of
 days.</p></dd><dt class="param">lastDate</dt><dd class="cmt"><p>the last date of the range for which to egt the nbr of
-days.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format of the provided dates</p></dd><dt>returns</dt><dd class="cmt"><p>the nbr of days between the two given dates</p></dd></dl></div>
+days.</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format of the provided dates</p></dd><dt>returns</dt><dd class="cmt"><p>the nbr of days between the two given dates</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#nbrOfDaysSince" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="nbrOfDaysSince(date:String,format:String):Int"></a>
       <a id="nbrOfDaysSince(String,String):Int"></a>
@@ -534,7 +677,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nbrOfDaysSince</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+        <span class="name">nbrOfDaysSince</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nbrOfDaysSince(date:String,format:String):Int" title="Permalink" target="_top">
@@ -543,7 +686,7 @@ <h4 class="signature">
     </span>
       <p class="shortcomment cmt">Returns the nbr of days between today and the given date.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the nbr of days between today and the given date.</p><pre><span class="cmt">// If today is "20170327":</span>
 assert(DateHelper.nbrOfDaysSince(<span class="lit">"20170310"</span>) == <span class="num">17</span>)
-assert(DateHelper.nbrOfDaysSince(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="num">17</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the nbr of days of diff with today</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format of the provided date</p></dd><dt>returns</dt><dd class="cmt"><p>the nbr of days between today and the given date</p></dd></dl></div>
+assert(DateHelper.nbrOfDaysSince(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="num">17</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the nbr of days of diff with today</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format of the provided date</p></dd><dt>returns</dt><dd class="cmt"><p>the nbr of days between today and the given date</p></dd></dl></div>
     </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="ne(x$1:AnyRef):Boolean"></a>
       <a id="ne(AnyRef):Boolean"></a>
@@ -570,17 +713,15 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">nextDay</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">nextDay</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@nextDay(date:String,format:String):String" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns for a date the date one day latter.</p><div class="fullcomment"><div class="comment cmt"><p>Returns for a date the date one day latter.</p><pre><span class="cmt">// If the given date is "20170310":</span>
-assert(DateHelper.nextDay(<span class="lit">"20170310"</span>) == <span class="lit">"20170311"</span>)
-assert(DateHelper.nextDay(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170311"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the date of the day after</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format of the provided and the
-returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date of the day after the given date</p></dd></dl></div>
+      <p class="shortcomment cmt">Returns for a date the date one day latter.</p><div class="fullcomment"><div class="comment cmt"><p>Returns for a date the date one day latter.</p><pre>assert(DateHelper.nextDay(<span class="lit">"20170310"</span>) == <span class="lit">"20170311"</span>)
+assert(DateHelper.nextDay(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170311"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the date of the day after</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format of the provided and the returned dates</p></dd><dt>returns</dt><dd class="cmt"><p>the date of the day after the given date</p></dd></dl></div>
     </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="notify():Unit"></a>
       <a id="notify():Unit"></a>
@@ -643,17 +784,15 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">previousDay</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">previousDay</span><span class="params">(<span name="date">date: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">defaultFormat</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@previousDay(date:String,format:String):String" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns for a date the date one day before.</p><div class="fullcomment"><div class="comment cmt"><p>Returns for a date the date one day before.</p><pre><span class="cmt">// If the given date is "20170310":</span>
-assert(DateHelper.previousDay(<span class="lit">"20170310"</span>) == <span class="lit">"20170309"</span>)
-assert(DateHelper.previousDay(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170309"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the date of the day before</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format of the provided and the
-returned dates.</p></dd><dt>returns</dt><dd class="cmt"><p>the date of the day before the given date</p></dd></dl></div>
+      <p class="shortcomment cmt">Returns for a date the date one day before.</p><div class="fullcomment"><div class="comment cmt"><p>Returns for a date the date one day before.</p><pre>assert(DateHelper.previousDay(<span class="lit">"20170310"</span>) == <span class="lit">"20170309"</span>)
+assert(DateHelper.previousDay(<span class="lit">"170310"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170309"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date for which to find the date of the day before</p></dd><dt class="param">format</dt><dd class="cmt"><p>the format of the provided and the returned dates</p></dd><dt>returns</dt><dd class="cmt"><p>the date of the day before the given date</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#reformatDate" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="reformatDate(date:String,inputFormat:String,outputFormat:String):String"></a>
       <a id="reformatDate(String,String,String):String"></a>
@@ -671,6 +810,29 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Reformats a date from one format to another.</p><div class="fullcomment"><div class="comment cmt"><p>Reformats a date from one format to another.</p><pre>assert(DateHelper.reformatDate(<span class="lit">"20170327"</span>, <span class="lit">"yyyyMMdd"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170327"</span>)</pre></div><dl class="paramcmts block"><dt class="param">date</dt><dd class="cmt"><p>the date to reformat</p></dd><dt class="param">inputFormat</dt><dd class="cmt"><p>the format in which the date to reformat is provided</p></dd><dt class="param">outputFormat</dt><dd class="cmt"><p>the format in which to format the provided date</p></dd><dt>returns</dt><dd class="cmt"><p>the date under the new format</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#setFormat" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="setFormat(format:String):Unit"></a>
+      <a id="setFormat(String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">setFormat</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@setFormat(format:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Sets the default date format used by these functions when no date format
+is specified.</p><div class="fullcomment"><div class="comment cmt"><p>Sets the default date format used by these functions when no date format
+is specified.</p><pre><span class="cmt">// By default, yyyyMMdd is used:</span>
+assert(<span class="num">3.</span>daysBefore == <span class="lit">"20170307"</span>)
+<span class="cmt">// But this can be modified globally:</span>
+DateHelper.setFormat(<span class="lit">"ddMMMyy"</span>)
+assert(<span class="num">3.</span>daysBefore == <span class="lit">"07Mar17"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>the new default format</p></dd></dl></div>
     </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
       <a id="synchronized[T0](⇒T0):T0"></a>
@@ -705,6 +867,24 @@ <h4 class="signature">
       </a>
     </span>
       <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#today" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="today:String"></a>
+      <a id="today:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">today</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@today:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns today's date/time under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns today's date/time under the default format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.today() == <span class="lit">"20170310"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>today's date under the default format</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#today" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="today(format:String):String"></a>
       <a id="today(String):String"></a>
@@ -714,7 +894,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">today</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">today</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@today(format:String):String" title="Permalink" target="_top">
@@ -722,8 +902,25 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Returns today's date/time under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns today's date/time under the requested format.</p><pre><span class="cmt">// If today's "20170310":</span>
-assert(DateHelper.today() == <span class="lit">"20170310"</span>)
-assert(DateHelper.today(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format for the current date</p></dd><dt>returns</dt><dd class="cmt"><p>today's date under the requested format</p></dd></dl></div>
+assert(DateHelper.today(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170310"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>the format for the current date</p></dd><dt>returns</dt><dd class="cmt"><p>today's date under the requested format</p></dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#twoDaysAgo" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="twoDaysAgo():String"></a>
+      <a id="twoDaysAgo():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">twoDaysAgo</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@twoDaysAgo():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns which date it was 2 days before today under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was 2 days before today under the default format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.twoDaysAgo() == <span class="lit">"20170308"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>the date of two days ago under the default format</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#twoDaysAgo" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="twoDaysAgo(format:String):String"></a>
       <a id="twoDaysAgo(String):String"></a>
@@ -733,7 +930,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">twoDaysAgo</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">twoDaysAgo</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@twoDaysAgo(format:String):String" title="Permalink" target="_top">
@@ -741,9 +938,7 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Returns which date it was 2 days before today under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns which date it was 2 days before today under the requested format.</p><pre><span class="cmt">// If today's "20170310":</span>
-assert(DateHelper.twoDaysAgo() == <span class="lit">"20170308"</span>)
-assert(DateHelper.twoDaysAgo(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170308"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format in which to output the
-date of two days ago.</p></dd><dt>returns</dt><dd class="cmt"><p>the date of two days ago under the requested format</p></dd></dl></div>
+assert(DateHelper.twoDaysAgo(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170308"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>the format in which to output the date of two days ago</p></dd><dt>returns</dt><dd class="cmt"><p>the date of two days ago under the requested format</p></dd></dl></div>
     </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="wait():Unit"></a>
       <a id="wait():Unit"></a>
@@ -813,6 +1008,24 @@ <h4 class="signature">
     </span>)</span>
               
         </dd></dl></div>
+    </li><li name="com.spark_helper.DateHelper#yesterday" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="yesterday:String"></a>
+      <a id="yesterday:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">yesterday</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.DateHelper$@yesterday:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Returns yesterday's date/time under the default format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns yesterday's date/time under the default format.</p><pre><span class="cmt">// If today's "20170310":</span>
+assert(DateHelper.yesterday() == <span class="lit">"20170309"</span>)</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>yesterday's date under the default format</p></dd></dl></div>
     </li><li name="com.spark_helper.DateHelper#yesterday" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="yesterday(format:String):String"></a>
       <a id="yesterday(String):String"></a>
@@ -822,7 +1035,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">yesterday</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;yyyyMMdd&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+        <span class="name">yesterday</span><span class="params">(<span name="format">format: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.DateHelper$@yesterday(format:String):String" title="Permalink" target="_top">
@@ -830,9 +1043,7 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Returns yesterday's date/time under the requested format.</p><div class="fullcomment"><div class="comment cmt"><p>Returns yesterday's date/time under the requested format.</p><pre><span class="cmt">// If today's "20170310":</span>
-assert(DateHelper.yesterday() == <span class="lit">"20170309"</span>)
-assert(DateHelper.yesterday(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170309"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format in which to output the
-date of yesterday.</p></dd><dt>returns</dt><dd class="cmt"><p>yesterday's date under the requested format</p></dd></dl></div>
+assert(DateHelper.yesterday(<span class="lit">"yyMMdd"</span>) == <span class="lit">"170309"</span>)</pre></div><dl class="paramcmts block"><dt class="param">format</dt><dd class="cmt"><p>the format in which to output the date of yesterday</p></dd><dt>returns</dt><dd class="cmt"><p>yesterday's date under the requested format</p></dd></dl></div>
     </li></ol>
             </div>
 
diff --git a/docs/com/spark_helper/HdfsHelper$$SeqExtensions.html b/docs/com/spark_helper/HdfsHelper$$SeqExtensions.html
new file mode 100644
index 0000000..b234a92
--- /dev/null
+++ b/docs/com/spark_helper/HdfsHelper$$SeqExtensions.html
@@ -0,0 +1,539 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>SeqExtensions - com.spark_helper.HdfsHelper.SeqExtensions</title>
+          <meta name="description" content="SeqExtensions - com.spark helper.HdfsHelper.SeqExtensions" />
+          <meta name="keywords" content="SeqExtensions com.spark helper.HdfsHelper.SeqExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.HdfsHelper$$SeqExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a></p>
+        <h1>SeqExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">package HdfsHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SeqExtensions</span><span class="tparams">[<span name="T">T &lt;: <span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.HdfsHelper.SeqExtensions"><span>SeqExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.HdfsHelper.SeqExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(seq:T)(implicitevidence$1:scala.reflect.ClassTag[T]):com.spark_helper.HdfsHelper.SeqExtensions[T]"></a>
+      <a id="&lt;init&gt;:SeqExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SeqExtensions</span><span class="params">(<span name="seq">seq: <span class="extype" name="com.spark_helper.HdfsHelper.SeqExtensions.T">T</span></span>)</span><span class="params">(<span class="implicit">implicit </span><span name="arg0">arg0: <span class="extype" name="scala.reflect.ClassTag">ClassTag</span>[<span class="extype" name="com.spark_helper.HdfsHelper.SeqExtensions.T">T</span>]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@&lt;init&gt;(seq:T)(implicitevidence$1:scala.reflect.ClassTag[T]):com.spark_helper.HdfsHelper.SeqExtensions[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper.SeqExtensions#seq" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="seq:T"></a>
+      <a id="seq:T"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">seq</span><span class="result">: <span class="extype" name="com.spark_helper.HdfsHelper.SeqExtensions.T">T</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@seq:T" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper.SeqExtensions#writeToHdfs" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="writeToHdfs(filePath:String):Unit"></a>
+      <a id="writeToHdfs(String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">writeToHdfs</span><span class="params">(<span name="filePath">filePath: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$SeqExtensions@writeToHdfs(filePath:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves list elements in a file on hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Saves list elements in a file on hdfs.</p><p>Please only consider this way of storing data when the data set is small
+enough.</p><p>Overwrites the file if it already exists.</p><pre><span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>).writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="std">List</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>).writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file in which to write the content of
+the List.</p></dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/HdfsHelper$$StringExtensions.html b/docs/com/spark_helper/HdfsHelper$$StringExtensions.html
new file mode 100644
index 0000000..ca7faf5
--- /dev/null
+++ b/docs/com/spark_helper/HdfsHelper$$StringExtensions.html
@@ -0,0 +1,536 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>StringExtensions - com.spark_helper.HdfsHelper.StringExtensions</title>
+          <meta name="description" content="StringExtensions - com.spark helper.HdfsHelper.StringExtensions" />
+          <meta name="keywords" content="StringExtensions com.spark helper.HdfsHelper.StringExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.HdfsHelper$$StringExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a></p>
+        <h1>StringExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">package HdfsHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.HdfsHelper.StringExtensions"><span>StringExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.HdfsHelper.StringExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(string:String):com.spark_helper.HdfsHelper.StringExtensions"></a>
+      <a id="&lt;init&gt;:StringExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringExtensions</span><span class="params">(<span name="string">string: <span class="extype" name="scala.Predef.String">String</span></span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@&lt;init&gt;(string:String):com.spark_helper.HdfsHelper.StringExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper.StringExtensions#string" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="string:String"></a>
+      <a id="string:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">string</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@string:String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper.StringExtensions#writeToHdfs" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="writeToHdfs(filePath:String):Unit"></a>
+      <a id="writeToHdfs(String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">writeToHdfs</span><span class="params">(<span name="filePath">filePath: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$$StringExtensions@writeToHdfs(filePath:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves the String in a file on hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Saves the String in a file on hdfs.</p><p>Overwrites the file if it already exists.</p><pre><span class="lit">"some\nrelatively small\ntext"</span>.writeToHdfsFile(<span class="lit">"/some/hdfs/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file in which to write the String</p></dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/HdfsHelper$.html b/docs/com/spark_helper/HdfsHelper$.html
index ab99d93..ee3a0ca 100644
--- a/docs/com/spark_helper/HdfsHelper$.html
+++ b/docs/com/spark_helper/HdfsHelper$.html
@@ -54,38 +54,39 @@ <h4 id="signature" class="signature">
 spark job and replace it with methods fully tested whose name is
 self-explanatory/readable.</p><p>For instance, one don't want to remove a file from hdfs using 3 lines of
 code and thus could instead just use
-HdfsHelper.deleteFile(&quot;my/hdfs/file/path.csv&quot;).</p><p>A few exemples:</p><pre><span class="kw">import</span> com.spark_helper.HdfsHelper
+HdfsHelper.deleteFile(&quot;my/hdfs/file/path.csv&quot;).</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.HdfsHelper
 
 <span class="cmt">// A bunch of methods wrapping the FileSystem API, such as:</span>
-HdfsHelper.fileExists(<span class="lit">"my/hdfs/file/path.txt"</span>)
-assert(HdfsHelper.listFileNamesInFolder(<span class="lit">"my/folder/path"</span>) == <span class="std">List</span>(<span class="lit">"file_name_1.txt"</span>, <span class="lit">"file_name_2.csv"</span>))
-assert(HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="lit">"20170306"</span>)
-assert(HdfsHelper.nbrOfDaysSinceFileWasLastModified(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="num">3</span>)
-HdfsHelper.deleteFile(<span class="lit">"my/hdfs/file/path.csv"</span>)
-HdfsHelper.moveFolder(<span class="lit">"my/hdfs/folder"</span>)
+HdfsHelper.fileExists(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// HdfsHelper.folderExists("my/hdfs/folder")</span>
+HdfsHelper.listFileNamesInFolder(<span class="lit">"my/folder/path"</span>) <span class="cmt">// List("file_name_1.txt", "file_name_2.csv")</span>
+HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// "20170306"</span>
+HdfsHelper.nbrOfDaysSinceFileWasLastModified(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// 3</span>
+HdfsHelper.deleteFile(<span class="lit">"my/hdfs/file/path.csv"</span>) <span class="cmt">// HdfsHelper.deleteFolder("my/hdfs/folder")</span>
+HdfsHelper.moveFolder(<span class="lit">"old/path"</span>, <span class="lit">"new/path"</span>) <span class="cmt">// HdfsHelper.moveFile("old/path.txt", "new/path.txt")</span>
+HdfsHelper.createEmptyHdfsFile(<span class="lit">"/some/hdfs/file/path.token"</span>) <span class="cmt">// HdfsHelper.createFolder("my/hdfs/folder")</span>
+
+<span class="cmt">// File content helpers:</span>
 HdfsHelper.compressFile(<span class="lit">"hdfs/path/to/uncompressed_file.txt"</span>, classOf[GzipCodec])
 HdfsHelper.appendHeader(<span class="lit">"my/hdfs/file/path.csv"</span>, <span class="lit">"colum0,column1"</span>)
 
 <span class="cmt">// Some Xml/Typesafe helpers for hadoop as well:</span>
-HdfsHelper.isHdfsXmlCompliantWithXsd(
-  <span class="lit">"my/hdfs/file/path.xml"</span>, getClass.getResource(<span class="lit">"/some_xml.xsd"</span>))
+HdfsHelper.isHdfsXmlCompliantWithXsd(<span class="lit">"my/hdfs/file/path.xml"</span>, getClass.getResource(<span class="lit">"/some_xml.xsd"</span>))
 HdfsHelper.loadXmlFileFromHdfs(<span class="lit">"my/hdfs/file/path.xml"</span>)
 
-<span class="cmt">// Very handy to load a config (typesafe format) stored on hdfs at the</span>
-<span class="cmt">// begining of a spark job:</span>
+<span class="cmt">// Very handy to load a config (typesafe format) stored on hdfs at the beginning of a spark job:</span>
 HdfsHelper.loadTypesafeConfigFromHdfs(<span class="lit">"my/hdfs/file/path.conf"</span>): Config
 
-<span class="cmt">// In order to write small amount of data in a file on hdfs without the</span>
-<span class="cmt">// whole spark stack:</span>
-HdfsHelper.writeToHdfsFile(
-  <span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>),
-  <span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="cmt">// In order to write small amount of data in a file on hdfs without the whole spark stack:</span>
+HdfsHelper.writeToHdfsFile(<span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>), <span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="cmt">// or:</span>
+<span class="kw">import</span> com.spark_helper.HdfsHelper._
+<span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>).writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="lit">"hello world"</span>.writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)
 
-<span class="cmt">// Deletes all files/folders in "hdfs/path/to/folder" for which the</span>
-<span class="cmt">// timestamp is older than 10 days:</span>
+<span class="cmt">// Deletes all files/folders in "hdfs/path/to/folder" for which the timestamp is older than 10 days:</span>
 HdfsHelper.purgeFolder(<span class="lit">"hdfs/path/to/folder"</span>, <span class="num">10</span>)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/HdfsHelper.scala">HdfsHelper</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl><div class="toggleContainer block">
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>Create a touch method</p></span></dd></dl><div class="toggleContainer block">
           <span class="toggle">Linear Supertypes</span>
           <div class="superTypes hiddenContent"><span class="extype" name="scala.Serializable">Serializable</span>, <span class="extype" name="java.io.Serializable">Serializable</span>, <span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
         </div></div>
@@ -124,7 +125,44 @@ <h4 id="signature" class="signature">
         <div id="allMembers">
         
 
-        
+        <div id="types" class="types members">
+              <h3>Type Members</h3>
+              <ol><li name="com.spark_helper.HdfsHelper.SeqExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SeqExtensions[T&lt;:Seq[String]]extendsAnyRef"></a>
+      <a id="SeqExtensions[T&lt;:Seq[String]]:SeqExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="HdfsHelper$$SeqExtensions.html"><span class="name">SeqExtensions</span></a><span class="tparams">[<span name="T">T &lt;: <span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$@SeqExtensions[T&lt;:Seq[String]]extendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.HdfsHelper.StringExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="StringExtensionsextendsAnyRef"></a>
+      <a id="StringExtensions:StringExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="HdfsHelper$$StringExtensions.html"><span class="name">StringExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$@StringExtensionsextendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
 
         
 
@@ -200,7 +238,7 @@ <h4 class="signature">
       <p class="shortcomment cmt">Appends a footer to a file.</p><div class="fullcomment"><div class="comment cmt"><p>Appends a footer to a file.</p><p>If the workingFolderPath parameter is provided, then the processing is
 done in a working/tmp folder and then only, the final file is moved to its
 final real location. This way, in case of cluster instability, i.e. in
-case the Spark job is interupted, this avoids having a temporary or
+case the Spark job is interrupted, this avoids having a temporary or
 corrupted file in output.
 </p></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file for which to add the footer</p></dd><dt class="param">footer</dt><dd class="cmt"><p>the footer to add</p></dd><dt class="param">workingFolderPath</dt><dd class="cmt"><p>the path where file manipulations will happen</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#appendHeader" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
@@ -219,11 +257,11 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Appends a header to a file.</p><div class="fullcomment"><div class="comment cmt"><p>Appends a header to a file.</p><p>Usefull when creating a csv file with spark and you need to add a header
+      <p class="shortcomment cmt">Appends a header to a file.</p><div class="fullcomment"><div class="comment cmt"><p>Appends a header to a file.</p><p>Useful when creating a csv file with spark and you need to add a header
 describing the different fields.</p><p>If the workingFolderPath parameter is provided, then the processing is
 done in a working/tmp folder and then only, the final file is moved to its
 final real location. This way, in case of cluster instability, i.e. in
-case the Spark job is interupted, this avoids having a temporary or
+case the Spark job is interrupted, this avoids having a temporary or
 corrupted file in output.
 </p></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file for which to add the header</p></dd><dt class="param">header</dt><dd class="cmt"><p>the header to add</p></dd><dt class="param">workingFolderPath</dt><dd class="cmt"><p>the path where file manipulations will happen</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#appendHeaderAndFooter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
@@ -242,11 +280,11 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Appends a header and a footer to a file.</p><div class="fullcomment"><div class="comment cmt"><p>Appends a header and a footer to a file.</p><p>Usefull when creating an xml file with spark and you need to add top level
+      <p class="shortcomment cmt">Appends a header and a footer to a file.</p><div class="fullcomment"><div class="comment cmt"><p>Appends a header and a footer to a file.</p><p>Useful when creating an xml file with spark and you need to add top level
 tags.</p><p>If the workingFolderPath parameter is provided, then the processing is
 done in a working/tmp folder and then only, the final file is moved to its
 final real location. This way, in case of cluster instability, i.e. in
-case the Spark job is interupted, this avoids having a temporary or
+case the Spark job is interrupted, this avoids having a temporary or
 corrupted file in output.
 </p></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file for which to add the header and the
 footer.</p></dd><dt class="param">header</dt><dd class="cmt"><p>the header to add</p></dd><dt class="param">footer</dt><dd class="cmt"><p>the footer to add</p></dd><dt class="param">workingFolderPath</dt><dd class="cmt"><p>the path where file manipulations will happen</p></dd></dl></div>
@@ -329,8 +367,8 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Creates an empty file on hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Creates an empty file on hdfs.</p><p>Might be usefull for token files. For instance a file which is only used
-as a timestamp token of the last update of a processus, or a file which
+      <p class="shortcomment cmt">Creates an empty file on hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Creates an empty file on hdfs.</p><p>Might be useful for token files. For instance a file which is only used
+as a timestamp token of the last update of a process, or a file which
 blocks the execution of an other instance of the same job, ...</p><p>Overwrites the file if it already exists.</p><pre>HdfsHelper.createEmptyHdfsFile(<span class="lit">"/some/hdfs/file/path.token"</span>)</pre><p>In case this is used as a timestamp container, you can then use the
 following methods to retrieve its timestamp:</p><pre><span class="kw">val</span> fileAge = HdfsHelper.nbrOfDaysSinceFileWasLastModified(<span class="lit">"/some/hdfs/file/path.token"</span>)
 <span class="kw">val</span> lastModificationDate = HdfsHelper.folderModificationDate(<span class="lit">"/some/hdfs/file/path.token"</span>)</pre></div><dl class="paramcmts block"><dt class="param">filePath</dt><dd class="cmt"><p>the path of the empty file to create</p></dd></dl></div>
@@ -456,9 +494,9 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns the stringified date of the last modification of the given file.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the stringified date of the last modification of the given file.</p><pre>assert(HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="lit">"20170306"</span>)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the file for which to get the last
+      <p class="shortcomment cmt">Returns the formatted date of the last modification of the given file.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the formatted date of the last modification of the given file.</p><pre>assert(HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="lit">"20170306"</span>)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the file for which to get the last
 modification date.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format under which to get the
-modification date.</p></dd><dt>returns</dt><dd class="cmt"><p>the stringified date of the last modification of the given file,
+modification date.</p></dd><dt>returns</dt><dd class="cmt"><p>the formatted date of the last modification of the given file,
 under the provided format.</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#fileModificationDateTime" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="fileModificationDateTime(hdfsPath:String):org.joda.time.DateTime"></a>
@@ -536,9 +574,9 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Returns the stringified date of the last modification of the given folder.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the stringified date of the last modification of the given folder.</p><pre>assert(HdfsHelper.folderModificationDate(<span class="lit">"my/hdfs/folder"</span>) == <span class="lit">"20170306"</span>)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the folder for which to get the last
+      <p class="shortcomment cmt">Returns the formatted date of the last modification of the given folder.</p><div class="fullcomment"><div class="comment cmt"><p>Returns the formatted date of the last modification of the given folder.</p><pre>assert(HdfsHelper.folderModificationDate(<span class="lit">"my/hdfs/folder"</span>) == <span class="lit">"20170306"</span>)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the folder for which to get the last
 modification date.</p></dd><dt class="param">format</dt><dd class="cmt"><p>(default = &quot;yyyyMMdd&quot;) the format under which to get the
-modification date.</p></dd><dt>returns</dt><dd class="cmt"><p>the stringified date of the last modification of the given folder,
+modification date.</p></dd><dt>returns</dt><dd class="cmt"><p>the formatted date of the last modification of the given folder,
 under the provided format.</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#folderModificationDateTime" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="folderModificationDateTime(hdfsPath:String):org.joda.time.DateTime"></a>
@@ -683,7 +721,7 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">Loads a typesafe config from Hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Loads a typesafe config from Hdfs.</p><p>The best way to load the configuration of your job from hdfs.</p><p>Typesafe is a config format which looks like this:</p><pre>config {
+      <p class="shortcomment cmt">Loads a Typesafe config from Hdfs.</p><div class="fullcomment"><div class="comment cmt"><p>Loads a Typesafe config from Hdfs.</p><p>The best way to load the configuration of your job from hdfs.</p><p>Typesafe is a config format which looks like this:</p><pre>config {
   airlines = [
     {
       code = QF
@@ -702,8 +740,8 @@ <h4 class="signature">
       }
     }
   ]
-}</pre></div><dl class="paramcmts block"><dt class="param">hdfsConfigPath</dt><dd class="cmt"><p>the absolute path of the typesafe config file on
-hdfs we want to load as a typesafe Config object.</p></dd><dt>returns</dt><dd class="cmt"><p>the com.typesafe.config.Config object which contains usable data</p></dd></dl></div>
+}</pre></div><dl class="paramcmts block"><dt class="param">hdfsConfigPath</dt><dd class="cmt"><p>the absolute path of the Typesafe config file on
+hdfs we want to load as a Typesafe Config object.</p></dd><dt>returns</dt><dd class="cmt"><p>the com.typesafe.config.Config object which contains usable data</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#loadXmlFileFromHdfs" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="loadXmlFileFromHdfs(hdfsXmlPath:String):scala.xml.Elem"></a>
       <a id="loadXmlFileFromHdfs(String):Elem"></a>
@@ -854,6 +892,50 @@ <h4 class="signature">
 <span class="cmt">// timestamp is older than 10 days:</span>
 HdfsHelper.purgeFolder(<span class="lit">"hdfs/path/to/folder"</span>, <span class="num">10</span>)</pre></div><dl class="paramcmts block"><dt class="param">folderPath</dt><dd class="cmt"><p>the path of the folder on hdfs to purge</p></dd><dt class="param">purgeAge</dt><dd class="cmt"><p>the threshold (in nbr of days) above which a file is
 considered too old and thus deleted/purged.</p></dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper#setConf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="setConf(configuration:org.apache.hadoop.conf.Configuration):Unit"></a>
+      <a id="setConf(Configuration):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">setConf</span><span class="params">(<span name="configuration">configuration: <span class="extype" name="org.apache.hadoop.conf.Configuration">Configuration</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$@setConf(configuration:org.apache.hadoop.conf.Configuration):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Configuration</code>
+used by the underlying <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+in case it requires some specificities.</code></code></p><div class="fullcomment"><div class="comment cmt"><p>Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Configuration</code>
+used by the underlying <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+in case it requires some specificities.</p><p>If this setter is not used, the default Configuration is set with
+<code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">new Configuration()</code>.
+</p></div><dl class="paramcmts block"><dt class="param">configuration</dt><dd class="cmt"><p>the specific Configuration to use</p></dd></dl></div>
+    </li><li name="com.spark_helper.HdfsHelper#setFileSystem" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="setFileSystem(fileSystem:org.apache.hadoop.fs.FileSystem):Unit"></a>
+      <a id="setFileSystem(FileSystem):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">setFileSystem</span><span class="params">(<span name="fileSystem">fileSystem: <span class="extype" name="org.apache.hadoop.fs.FileSystem">FileSystem</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.HdfsHelper$@setFileSystem(fileSystem:org.apache.hadoop.fs.FileSystem):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+in case it requires some specificities.</code></p><div class="fullcomment"><div class="comment cmt"><p>Sets a specific <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem</code>
+in case it requires some specificities.</p><p>If this setter is not used, the default FileSystem is set with
+<code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileSystem.get(new Configuration())</code>.
+</p></div><dl class="paramcmts block"><dt class="param">fileSystem</dt><dd class="cmt"><p>the specific FileSystem to use</p></dd></dl></div>
     </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
       <a id="synchronized[T0](⇒T0):T0"></a>
@@ -999,7 +1081,7 @@ <h4 class="signature">
 enough.</p><p>Overwrites the file if it already exists.</p><pre>HdfsHelper.writeToHdfsFile(
   <span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>), <span class="lit">"/some/hdfs/file/path.txt"</span>)
 HdfsHelper.writeToHdfsFile(
-  <span class="std">List</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>), <span class="lit">"/some/hdfs/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">content</dt><dd class="cmt"><p>the array of strings to write in the file as one line per
+  <span class="std">List</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>), <span class="lit">"/some/hdfs/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">content</dt><dd class="cmt"><p>the seq of strings to write in the file as one line per
 string (this takes care of joining strings with &quot;\n&quot;s).</p></dd><dt class="param">filePath</dt><dd class="cmt"><p>the path of the file in which to write the content</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper#writeToHdfsFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="writeToHdfsFile(content:String,filePath:String):Unit"></a>
diff --git a/docs/com/spark_helper/Monitor$.html b/docs/com/spark_helper/Monitor$.html
index 955f476..15e0a20 100644
--- a/docs/com/spark_helper/Monitor$.html
+++ b/docs/com/spark_helper/Monitor$.html
@@ -49,11 +49,11 @@ <h4 id="signature" class="signature">
       </span>
       </h4>
       
-          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>A logger dedicated to Spak jobs.</p><p>It's a simple logger/report which contains a report that one can update from
+          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>A logger dedicated to Spark jobs.</p><p>It's a simple logger/report which contains a report that one can update from
 the driver and a success state. The idea is to persist job executions logs
-and errors (and forget about grepping unreadable yarn logs).</p><p>It's designed for perdiodic spark jobs (handles storage and purge of logs)
+and errors (and forget about grepping unreadable yarn logs).</p><p>It's designed for periodic spark jobs (handles storage and purge of logs)
 and provides a way to handle kpis validation.</p><p>Logs are stored on the go which means one can have a direct real time access
-of the job logs/status and it's current state (which can overwise be a pain
+of the job logs/status and it's current state (which can otherwise be a pain
 if it means going through yarn logs, or even for certain production
 environments going through additional layers of software logs to get to yarn
 logs).</p><p>One of the issues this logger aims at tackling is the handling of exceptions
@@ -62,9 +62,9 @@ <h4 id="signature" class="signature">
 want to perform a few actions before letting the job crash. The idea is thus
 to surround (driver side) a Spark pipeline within a try catch and redirect
 the exception to the logger for a clean logging.</p><p>This is a &quot;driver-only&quot; logger and is not intended at logging concurrent
-actions from executors.</p><p>Produced reports can easily be inserted in a notification email whenerver
+actions from executors.</p><p>Produced reports can easily be inserted in a notification email whenever
 the job fails, which saves a lot of time to maintainers operating on heavy
-production environements.</p><p>The produced persisted report is also a way for downstream jobs to know the
+production environments.</p><p>The produced persisted report is also a way for downstream jobs to know the
 status of their input data.</p><p>Let's go through a simple Spark job example monitored with this Monitor
 facility:</p><pre>Monitor.setTitle(<span class="lit">"My job title"</span>)
 Monitor.addDescription(
@@ -83,7 +83,7 @@ <h4 id="signature" class="signature">
       Test(<span class="lit">"Nbr of output records"</span>, processedData.count(), SUPERIOR_THAN, <span class="num">10</span>e6d, NBR),
       Test(<span class="lit">"Some pct of invalid output"</span>, your_complex_kpi, INFERIOR_THAN, <span class="num">3</span>, PCT)
     ),
-    <span class="lit">"My pipeline descirption"</span>
+    <span class="lit">"My pipeline description"</span>
   )
 
   <span class="kw">if</span> (outputIsValid)
@@ -91,9 +91,9 @@ <h4 id="signature" class="signature">
 
 } <span class="kw">catch</span> {
   <span class="kw">case</span> iie: InvalidInputException <span class="kw">=&gt;</span>
-    Monitor.error(iie, <span class="lit">"My pipeline descirption"</span>, diagnostic = <span class="lit">"No input data!"</span>)
+    Monitor.error(iie, <span class="lit">"My pipeline description"</span>, diagnostic = <span class="lit">"No input data!"</span>)
   <span class="kw">case</span> e: Throwable <span class="kw">=&gt;</span>
-    Monitor.error(e, <span class="lit">"My pipeline descirption"</span>) <span class="cmt">// whatever unexpected error</span>
+    Monitor.error(e, <span class="lit">"My pipeline description"</span>) <span class="cmt">// whatever unexpected error</span>
 }
 
 <span class="kw">if</span> (Monitor.isSuccess()) {
@@ -105,7 +105,7 @@ <h4 id="signature" class="signature">
 <span class="cmt">// HDFS (this saves the logs in the folder set with Monitor.setLogFolder):</span>
 Monitor.store()
 
-<span class="cmt">// At the end of the job, if the job isn't successfull, you might want to</span>
+<span class="cmt">// At the end of the job, if the job isn't successful, you might want to</span>
 <span class="cmt">// crash it (for instance to get a notification from your scheduler):</span>
 <span class="kw">if</span> (!Monitor.isSuccess()) <span class="kw">throw</span> <span class="kw">new</span> Exception() <span class="cmt">// or send an email, or ...</span></pre><p>At any time during the job, logs can be accessed from file
 path/to/log/folder/current.ongoing</p><p>If we were to read the stored report after this simple pipeline, here are
@@ -113,8 +113,8 @@ <h4 id="signature" class="signature">
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">23</span>] My pipeline descirption: failed
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">23</span>] My pipeline description: failed
   Diagnostic: No input data!
     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs:<span class="cmt">//my/hdfs/input/path</span>
     at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:<span class="num">285</span>)
@@ -124,18 +124,18 @@ <h4 id="signature" class="signature">
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">36</span>] My pipeline descirption: failed
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">36</span>] My pipeline description: failed
     java.lang.NumberFormatException: For input string: <span class="lit">"a"</span>
     java.lang.NumberFormatException.forInputString(NumberFormatException.java:<span class="num">65</span>)
     java.lang.Integer.parseInt(Integer.java:<span class="num">492</span>)
     ...
-[<span class="num">10</span>:<span class="num">36</span>] Duration: <span class="num">00</span>:<span class="num">13</span>:<span class="num">47</span></pre><p>Another scenario, successfull spark pipeline and KPIs are valid; all good!:</p><pre>          My job title
+[<span class="num">10</span>:<span class="num">36</span>] Duration: <span class="num">00</span>:<span class="num">13</span>:<span class="num">47</span></pre><p>Another scenario, successful spark pipeline and KPIs are valid; all good!:</p><pre>          My job title
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">41</span>] My pipeline descirption: success
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">41</span>] My pipeline description: success
   KPI: Nbr of output records
     Value: <span class="num">14669071.0</span>
     Must be superior than <span class="num">10000000.0</span>
@@ -147,7 +147,7 @@ <h4 id="signature" class="signature">
 [<span class="num">10</span>:<span class="num">41</span>-<span class="num">10</span>:<span class="num">42</span>] My second pipeline description: success
 [<span class="num">10</span>:<span class="num">42</span>] Duration: <span class="num">00</span>:<span class="num">19</span>:<span class="num">23</span></pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/monitoring/Monitor.scala">Monitor</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl><div class="toggleContainer block">
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>would a State monad be appropriate?</p></span></dd></dl><div class="toggleContainer block">
           <span class="toggle">Linear Supertypes</span>
           <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
         </div></div>
@@ -262,7 +262,7 @@ <h4 class="signature">
       <p class="shortcomment cmt">Sets the report's contact list.</p><div class="fullcomment"><div class="comment cmt"><p>Sets the report's contact list.</p><p>This will appear within the first lines of the report:</p><pre><span class="cmt">// Using:</span>
 Monitor.setReportTitle(<span class="lit">"My Simple Job"</span>)
 Monitor.addContacts(<span class="std">List</span>(<span class="lit">"x.guihot@gmail.com"</span>, <span class="lit">"smbdy@gmail.com"</span>))
-<span class="cmt">// Produces this at the begining of the report:</span>
+<span class="cmt">// Produces this at the beginning of the report:</span>
 <span class="lit">"          My Simple Job"</span>
 <span class="lit">""</span>
 <span class="lit">"Point of contact: x.guihot@gmail.com, smbdy@gmail.com"</span></pre></div><dl class="paramcmts block"><dt class="param">contacts</dt><dd class="cmt"><p>the list of points of contact</p></dd></dl></div>
@@ -285,7 +285,7 @@ <h4 class="signature">
       <p class="shortcomment cmt">Sets the report's description.</p><div class="fullcomment"><div class="comment cmt"><p>Sets the report's description.</p><p>This will appear within the first lines of the report:</p><pre><span class="cmt">// Using:</span>
 Monitor.setReportTitle(<span class="lit">"My Simple Job"</span>)
 Monitor.addDescription(<span class="lit">"Documentation: https://github.com/xavierguihot/spark_helper"</span>)
-<span class="cmt">// Produces this at the begining of the report:</span>
+<span class="cmt">// Produces this at the beginning of the report:</span>
 <span class="lit">"          My Simple Job"</span>
 <span class="lit">""</span>
 <span class="lit">"Documentation: https://github.com/xavierguihot/spark_helper"</span></pre></div><dl class="paramcmts block"><dt class="param">description</dt><dd class="cmt"><p>the description of the Spark job (or whatever)</p></dd></dl></div>
@@ -372,7 +372,7 @@ <h4 class="signature">
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">error</span><span class="params">(<span name="exception">exception: <span class="extype" name="java.lang.Throwable">Throwable</span></span>, <span name="taskDescription">taskDescription: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="diagnostic">diagnostic: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+        <span class="name">error</span><span class="params">(<span name="exception">exception: <span class="extype" name="scala.Throwable">Throwable</span></span>, <span name="taskDescription">taskDescription: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="diagnostic">diagnostic: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;&quot;</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
       </span>
       </h4><span class="permalink">
       <a href="../../index.html#com.spark_helper.Monitor$@error(exception:Throwable,taskDescription:String,diagnostic:String):Boolean" title="Permalink" target="_top">
@@ -385,8 +385,8 @@ <h4 class="signature">
 catch whatever exception from executors and thus log the exact error while
 still being able to keep on with the job or end it properly.</p><p>Catching an error like this:</p><pre>monitor.error(
   invalidInputException,
-  <span class="lit">"My pipeline descirption"</span>,
-  diagnostic = <span class="lit">"No input data!"</span>)</pre><p>will result in this to be appended to the report:</p><pre>[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">24</span>] My pipeline descirption: failed
+  <span class="lit">"My pipeline description"</span>,
+  diagnostic = <span class="lit">"No input data!"</span>)</pre><p>will result in this to be appended to the report:</p><pre>[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">24</span>] My pipeline description: failed
   Diagnostic: No input data!
     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs:<span class="cmt">//my/hdfs/input/path</span>
     at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:<span class="num">285</span>)
@@ -410,7 +410,7 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Updates the report with some text and a failure.</p><div class="fullcomment"><div class="comment cmt"><p>Updates the report with some text and a failure.</p><p>This sets the status of the monitoring to false. After that the status
-will never be success again, even if you update the report with success().</p><p>Using this method like this:</p><pre>monitor.error(<span class="lit">"Some text"</span>)</pre><p>will result in this to be appended to the report:</p><pre><span class="lit">"[10:35-10:37] Some text: failure\n"</span></pre><p>Once the monitoring is a failure, then whatever following successfull
+will never be success again, even if you update the report with success().</p><p>Using this method like this:</p><pre>monitor.error(<span class="lit">"Some text"</span>)</pre><p>will result in this to be appended to the report:</p><pre><span class="lit">"[10:35-10:37] Some text: failure\n"</span></pre><p>Once the monitoring is a failure, then whatever following successful
 action won't change the failed status of the monitoring.
 </p></div><dl class="paramcmts block"><dt class="param">taskDescription</dt><dd class="cmt"><p>the text to append to the report</p></dd><dt>returns</dt><dd class="cmt"><p>false since it's a failure</p></dd></dl></div>
     </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
@@ -488,18 +488,18 @@ <h4 class="signature">
     </span>
       <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
     </li><li name="com.spark_helper.Monitor#isSuccess" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="isSuccess():Boolean"></a>
-      <a id="isSuccess():Boolean"></a>
+      <a id="isSuccess:Boolean"></a>
+      <a id="isSuccess:Boolean"></a>
       <h4 class="signature">
       <span class="modifier_kind">
         <span class="modifier"></span>
         <span class="kind">def</span>
       </span>
       <span class="symbol">
-        <span class="name">isSuccess</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+        <span class="name">isSuccess</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
       </span>
       </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.Monitor$@isSuccess():Boolean" title="Permalink" target="_top">
+      <a href="../../index.html#com.spark_helper.Monitor$@isSuccess:Boolean" title="Permalink" target="_top">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
@@ -696,7 +696,7 @@ <h4 class="signature">
     </span>
       <p class="shortcomment cmt">Sets the report's title.</p><div class="fullcomment"><div class="comment cmt"><p>Sets the report's title.</p><p>This will be the first line of the report:</p><pre><span class="cmt">// Using:</span>
 Monitor.setReportTitle(<span class="lit">"My Simple Job"</span>)
-<span class="cmt">// Produces this at the begining of the report:</span>
+<span class="cmt">// Produces this at the beginning of the report:</span>
 <span class="lit">"          My Simple Job"</span>
 <span class="lit">""</span></pre></div><dl class="paramcmts block"><dt class="param">title</dt><dd class="cmt"><p>the title of the report</p></dd></dl></div>
     </li><li name="com.spark_helper.Monitor#store" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
diff --git a/docs/com/spark_helper/SparkHelper$$OptionRDDExtensions.html b/docs/com/spark_helper/SparkHelper$$OptionRDDExtensions.html
new file mode 100644
index 0000000..66a0d24
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$OptionRDDExtensions.html
@@ -0,0 +1,540 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>OptionRDDExtensions - com.spark_helper.SparkHelper.OptionRDDExtensions</title>
+          <meta name="description" content="OptionRDDExtensions - com.spark helper.SparkHelper.OptionRDDExtensions" />
+          <meta name="keywords" content="OptionRDDExtensions com.spark helper.SparkHelper.OptionRDDExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$OptionRDDExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>OptionRDDExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">OptionRDDExtensions</span><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.OptionRDDExtensions"><span>OptionRDDExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.OptionRDDExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[Option[T]])(implicitevidence$3:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.OptionRDDExtensions[T]"></a>
+      <a id="&lt;init&gt;:OptionRDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">OptionRDDExtensions</span><span class="params">(<span name="rdd">rdd: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Option">Option</span>[<span class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions.T">T</span>]]</span>)</span><span class="params">(<span class="implicit">implicit </span><span name="arg0">arg0: <span class="extype" name="scala.reflect.ClassTag">ClassTag</span>[<span class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions.T">T</span>]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[Option[T]])(implicitevidence$3:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.OptionRDDExtensions[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.OptionRDDExtensions#flatten" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="flatten:org.apache.spark.rdd.RDD[T]"></a>
+      <a id="flatten:RDD[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">flatten</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions.T">T</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@flatten:org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Option[T]</code>
+to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.</code></code></p><div class="fullcomment"><div class="comment cmt"><p>Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Option[T]</code>
+to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.</p><pre>sc.parallelize(<span class="std">Array</span>(<span class="std">Some</span>(<span class="num">1</span>), <span class="std">None</span>, <span class="std">Some</span>(<span class="num">2</span>))).flatten == sc.parallelize(<span class="std">Array</span>(<span class="std">Seq</span>(<span class="num">1</span>, <span class="num">2</span>)))</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>the flat RDD as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD.flatMap(x => x)</code>
+or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
+would have.</p></dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.OptionRDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="rdd:org.apache.spark.rdd.RDD[Option[T]]"></a>
+      <a id="rdd:RDD[Option[T]]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">rdd</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Option">Option</span>[<span class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions.T">T</span>]]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@rdd:org.apache.spark.rdd.RDD[Option[T]]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$OptionRDDExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$$PairRDDExtensions.html b/docs/com/spark_helper/SparkHelper$$PairRDDExtensions.html
new file mode 100644
index 0000000..b136443
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$PairRDDExtensions.html
@@ -0,0 +1,376 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>PairRDDExtensions - com.spark_helper.SparkHelper.PairRDDExtensions</title>
+          <meta name="description" content="PairRDDExtensions - com.spark helper.SparkHelper.PairRDDExtensions" />
+          <meta name="keywords" content="PairRDDExtensions com.spark helper.SparkHelper.PairRDDExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$PairRDDExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>PairRDDExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">PairRDDExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyVal">AnyVal</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.PairRDDExtensions"><span>PairRDDExtensions</span></li><li class="in" name="scala.AnyVal"><span>AnyVal</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.PairRDDExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[(String,String)]):com.spark_helper.SparkHelper.PairRDDExtensions"></a>
+      <a id="&lt;init&gt;:PairRDDExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">PairRDDExtensions</span><span class="params">(<span name="rdd">rdd: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[(String,String)]):com.spark_helper.SparkHelper.PairRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.Any#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyVal#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="scala.AnyVal">AnyVal</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@getClass():Class[_&lt;:AnyVal]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyVal → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="rdd:org.apache.spark.rdd.RDD[(String,String)]"></a>
+      <a id="rdd:RDD[(String,String)]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">rdd</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@rdd:org.apache.spark.rdd.RDD[(String,String)]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileByKey(path:String,keyNbr:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="saveAsTextFileByKey(String,Int,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="keyNbr">keyNbr: <span class="extype" name="scala.Int">Int</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@saveAsTextFileByKey(path:String,keyNbr:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided path, there will be one file per key in the given
+keyValueRDD. And within a file for a given key are only stored values
+for this key.</p><p>This is not scalable. This shouldn't be considered for any data flow
+with normal or big volumes.</p><pre>rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>, <span class="num">12</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will be stored key files</p></dd><dt class="param">keyNbr</dt><dd class="cmt"><p>the nbr of expected keys (which is the nbr of output
+files)</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileByKey(path:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="saveAsTextFileByKey(String,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@saveAsTextFileByKey(path:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided path, there will be one file per key in the given
+keyValueRDD. And within a file for a given key are only stored values
+for this key.</p><p>As this internally needs to know the nbr of keys, this will have to
+compute it. If this nbr of keys is known beforehand, it would spare
+resources to use
+<code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsTextFileByKey(path: String, keyNbr: Int, codec: Class[_ <: CompressionCodec])</code>
+instead.</p><p>This is not scalable. This shouldn't be considered for any data flow
+with normal or big volumes.</p><pre>rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will be stored key files</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileByKey(path:String,keyNbr:Int):Unit"></a>
+      <a id="saveAsTextFileByKey(String,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="keyNbr">keyNbr: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@saveAsTextFileByKey(path:String,keyNbr:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided path, there will be one file per key in the given
+keyValueRDD. And within a file for a given key are only stored values
+for this key.</p><p>This is not scalable. This shouldn't be considered for any data flow
+with normal or big volumes.</p><pre>rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>, <span class="num">12</span>)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will be stored key files</p></dd><dt class="param">keyNbr</dt><dd class="cmt"><p>the nbr of expected keys (which is the nbr of output
+files)</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileByKey(path:String):Unit"></a>
+      <a id="saveAsTextFileByKey(String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@saveAsTextFileByKey(path:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided path, there will be one file per key in the given
+keyValueRDD. And within a file for a given key are only stored values
+for this key.</p><p>As this internally needs to know the nbr of keys, this will have to
+compute it. If this nbr of keys is known beforehand, it would spare
+resources to use <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsTextFileByKey(path: String, keyNbr: Int)</code>
+instead.</p><p>This is not scalable. This shouldn't be considered for any data flow
+with normal or big volumes.</p><pre>rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will be stored key files</p></dd></dl></div>
+    </li><li name="scala.Any#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$PairRDDExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyVal">
+              <h3>Inherited from <span class="extype" name="scala.AnyVal">AnyVal</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$$RDDExtensions.html b/docs/com/spark_helper/SparkHelper$$RDDExtensions.html
new file mode 100644
index 0000000..cfdc746
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$RDDExtensions.html
@@ -0,0 +1,547 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>RDDExtensions - com.spark_helper.SparkHelper.RDDExtensions</title>
+          <meta name="description" content="RDDExtensions - com.spark helper.SparkHelper.RDDExtensions" />
+          <meta name="keywords" content="RDDExtensions com.spark helper.SparkHelper.RDDExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$RDDExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>RDDExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">RDDExtensions</span><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.RDDExtensions"><span>RDDExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.RDDExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[T])(implicitevidence$1:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.RDDExtensions[T]"></a>
+      <a id="&lt;init&gt;:RDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">RDDExtensions</span><span class="params">(<span name="rdd">rdd: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>)</span><span class="params">(<span class="implicit">implicit </span><span name="arg0">arg0: <span class="extype" name="scala.reflect.ClassTag">ClassTag</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[T])(implicitevidence$1:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.RDDExtensions[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.RDDExtensions#partialMap" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="partialMap(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]"></a>
+      <a id="partialMap(PartialFunction[T,T]):RDD[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">partialMap</span><span class="params">(<span name="pf">pf: <span class="extype" name="scala.PartialFunction">PartialFunction</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>, <span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@partialMap(pf:PartialFunction[T,T]):org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Map an RDD to the same type, by applying a partial function and the
+identity otherwise.</p><div class="fullcomment"><div class="comment cmt"><p>Map an RDD to the same type, by applying a partial function and the
+identity otherwise.</p><p>Avoids having <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">case x => x</code>.</p><p>Similar idea to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">.collect</code>,
+but instead of skipping non-matching items, it keeps them as-is.</p><pre>sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
+<span class="cmt">// is equivalent to:</span>
+sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">2</span>, <span class="num">7</span>, <span class="num">8</span>)).map {
+  <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a
+  <span class="kw">case</span> a               <span class="kw">=&gt;</span> a
+}
+<span class="cmt">// in order to map to:</span>
+sc.parallelize(<span class="std">Array</span>(<span class="num">1</span>, <span class="num">3</span>, <span class="num">4</span>, <span class="num">7</span>, <span class="num">16</span>))</pre></div><dl class="paramcmts block"><dt class="param">pf</dt><dd class="cmt"><p>the partial function to apply</p></dd><dt>returns</dt><dd class="cmt"><p>an rdd of the same type, for which each element is either the
+application of the partial function where defined or the identity.</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.RDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="rdd:org.apache.spark.rdd.RDD[T]"></a>
+      <a id="rdd:RDD[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">rdd</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.RDDExtensions.T">T</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@rdd:org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$RDDExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$$SeqRDDExtensions.html b/docs/com/spark_helper/SparkHelper$$SeqRDDExtensions.html
new file mode 100644
index 0000000..190d93d
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$SeqRDDExtensions.html
@@ -0,0 +1,540 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>SeqRDDExtensions - com.spark_helper.SparkHelper.SeqRDDExtensions</title>
+          <meta name="description" content="SeqRDDExtensions - com.spark helper.SparkHelper.SeqRDDExtensions" />
+          <meta name="keywords" content="SeqRDDExtensions com.spark helper.SparkHelper.SeqRDDExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$SeqRDDExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>SeqRDDExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SeqRDDExtensions</span><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.SeqRDDExtensions"><span>SeqRDDExtensions</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.SeqRDDExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[Seq[T]])(implicitevidence$2:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.SeqRDDExtensions[T]"></a>
+      <a id="&lt;init&gt;:SeqRDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SeqRDDExtensions</span><span class="params">(<span name="rdd">rdd: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions.T">T</span>]]</span>)</span><span class="params">(<span class="implicit">implicit </span><span name="arg0">arg0: <span class="extype" name="scala.reflect.ClassTag">ClassTag</span>[<span class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions.T">T</span>]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[Seq[T]])(implicitevidence$2:scala.reflect.ClassTag[T]):com.spark_helper.SparkHelper.SeqRDDExtensions[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@clone():Object" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SeqRDDExtensions#flatten" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="flatten:org.apache.spark.rdd.RDD[T]"></a>
+      <a id="flatten:RDD[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">flatten</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions.T">T</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@flatten:org.apache.spark.rdd.RDD[T]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Seq[T]</code>
+to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.</code></code></p><div class="fullcomment"><div class="comment cmt"><p>Flattens an RDD of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">Seq[T]</code>
+to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[T]</code>.</p><pre>sc.parallelize(<span class="std">Array</span>(<span class="std">Seq</span>(<span class="num">1</span>, <span class="num">2</span>, <span class="num">3</span>), Nil, <span class="std">Seq</span>(<span class="num">4</span>))).flatten == sc.parallelize(<span class="std">Array</span>(<span class="std">Seq</span>(<span class="num">1</span>, <span class="num">2</span>, <span class="num">3</span>, <span class="num">4</span>)))</pre></div><dl class="paramcmts block"><dt>returns</dt><dd class="cmt"><p>the flat RDD as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD.flatMap(identity)</code>
+or <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">List.flatten</code>
+would have.</p></dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@notify():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SeqRDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="rdd:org.apache.spark.rdd.RDD[Seq[T]]"></a>
+      <a id="rdd:RDD[Seq[T]]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">rdd</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions.T">T</span>]]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@rdd:org.apache.spark.rdd.RDD[Seq[T]]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@wait():Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SeqRDDExtensions@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$$SparkContextExtensions.html b/docs/com/spark_helper/SparkHelper$$SparkContextExtensions.html
new file mode 100644
index 0000000..cda2b45
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$SparkContextExtensions.html
@@ -0,0 +1,476 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>SparkContextExtensions - com.spark_helper.SparkHelper.SparkContextExtensions</title>
+          <meta name="description" content="SparkContextExtensions - com.spark helper.SparkHelper.SparkContextExtensions" />
+          <meta name="keywords" content="SparkContextExtensions com.spark helper.SparkHelper.SparkContextExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$SparkContextExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>SparkContextExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SparkContextExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyVal">AnyVal</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.SparkContextExtensions"><span>SparkContextExtensions</span></li><li class="in" name="scala.AnyVal"><span>AnyVal</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.SparkContextExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(sc:org.apache.spark.SparkContext):com.spark_helper.SparkHelper.SparkContextExtensions"></a>
+      <a id="&lt;init&gt;:SparkContextExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SparkContextExtensions</span><span class="params">(<span name="sc">sc: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@&lt;init&gt;(sc:org.apache.spark.SparkContext):com.spark_helper.SparkHelper.SparkContextExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.Any#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#decreaseCoalescence" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalesceLevel:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="decreaseCoalescence(String,String,Int,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">decreaseCoalescence</span><span class="params">(<span name="highCoalescenceLevelFolder">highCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lowerCoalescenceLevelFolder">lowerCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalesceLevel">finalCoalesceLevel: <span class="extype" name="scala.Int">Int</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalesceLevel:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Decreases the nbr of partitions of a folder.</p><div class="fullcomment"><div class="comment cmt"><p>Decreases the nbr of partitions of a folder.</p><p>This comes in handy when the last step of your job needs to run on
+thousands of files, but you want to store your final output on let's say
+only 30 files.</p><p>It's like a <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileUtil.copyMerge()</code>
+, but the merging produces more than one file.</p><p>Be aware that this methods deletes the provided input folder.</p><pre>sc.decreaseCoalescence(
+  <span class="lit">"/folder/path/with/2000/files"</span>,
+  <span class="lit">"/produced/folder/path/with/only/30/files"</span>,
+  <span class="num">30</span>,
+  classOf[BZip2Codec]
+)</pre></div><dl class="paramcmts block"><dt class="param">highCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which contains 10000 files</p></dd><dt class="param">lowerCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which will contain the same
+data as highCoalescenceLevelFolder but spread on only 30 files (where 30
+is the finalCoalesceLevel parameter).</p></dd><dt class="param">finalCoalesceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
+of this method.</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#decreaseCoalescence" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalesceLevel:Int):Unit"></a>
+      <a id="decreaseCoalescence(String,String,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">decreaseCoalescence</span><span class="params">(<span name="highCoalescenceLevelFolder">highCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lowerCoalescenceLevelFolder">lowerCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalesceLevel">finalCoalesceLevel: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalesceLevel:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Decreases the nbr of partitions of a folder.</p><div class="fullcomment"><div class="comment cmt"><p>Decreases the nbr of partitions of a folder.</p><p>This comes in handy when the last step of your job needs to run on
+thousands of files, but you want to store your final output on let's say
+only 30 files.</p><p>It's like a <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">FileUtil.copyMerge()</code>
+, but the merging produces more than one file.</p><p>Be aware that this methods deletes the provided input folder.</p><pre>sc.decreaseCoalescence(
+  <span class="lit">"/folder/path/with/2000/files"</span>,
+  <span class="lit">"/produced/folder/path/with/only/30/files"</span>,
+  <span class="num">30</span>
+)</pre></div><dl class="paramcmts block"><dt class="param">highCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which contains 10000 files</p></dd><dt class="param">lowerCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which will contain the same
+data as highCoalescenceLevelFolder but spread on only 30 files (where 30
+is the finalCoalesceLevel parameter).</p></dd><dt class="param">finalCoalesceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
+of this method.</p></dd></dl></div>
+    </li><li name="scala.AnyVal#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="scala.AnyVal">AnyVal</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@getClass():Class[_&lt;:AnyVal]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyVal → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#sc" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="sc:org.apache.spark.SparkContext"></a>
+      <a id="sc:SparkContext"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">sc</span><span class="result">: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@sc:org.apache.spark.SparkContext" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#textFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="textFile(paths:Seq[String],minPartitions:Int):org.apache.spark.rdd.RDD[String]"></a>
+      <a id="textFile(Seq[String],Int):RDD[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">textFile</span><span class="params">(<span name="paths">paths: <span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="minPartitions">minPartitions: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@textFile(paths:Seq[String],minPartitions:Int):org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+when files contains commas in their name.</code></p><div class="fullcomment"><div class="comment cmt"><p>A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+when files contains commas in their name.</p><p>As <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+allows to provide several files at once by giving them as a string which
+is a list of strings joined with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">,</code>,
+we can't give it files containing commas in their name.</p><p>This method aims at bypassing this limitation by passing paths as a
+sequence of strings.</p><pre>sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre></div><dl class="paramcmts block"><dt class="param">paths</dt><dd class="cmt"><p>the paths of the file(s)/folder(s) to read</p></dd><dt class="param">minPartitions</dt><dd class="cmt"><p>the nbr of partitions in which to split the input</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#textFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="textFile(paths:Seq[String]):org.apache.spark.rdd.RDD[String]"></a>
+      <a id="textFile(Seq[String]):RDD[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">textFile</span><span class="params">(<span name="paths">paths: <span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@textFile(paths:Seq[String]):org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+when files contains commas in their name.</code></p><div class="fullcomment"><div class="comment cmt"><p>A replacement for <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+when files contains commas in their name.</p><p>As <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sc.textFile()</code>
+allows to provide several files at once by giving them as a string which
+is a list of strings joined with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">,</code>,
+we can't give it files containing commas in their name.</p><p>This method aims at bypassing this limitation by passing paths as a
+sequence of strings.</p><pre>sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre></div><dl class="paramcmts block"><dt class="param">paths</dt><dd class="cmt"><p>the paths of the file(s)/folder(s) to read</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#textFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="textFile(path:String,delimiter:String,maxRecordLength:String):org.apache.spark.rdd.RDD[String]"></a>
+      <a id="textFile(String,String,String):RDD[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">textFile</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="delimiter">delimiter: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="maxRecordLength">maxRecordLength: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;1000000&quot;</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@textFile(path:String,delimiter:String,maxRecordLength:String):org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+, but for a specific record delimiter.</code></p><div class="fullcomment"><div class="comment cmt"><p>Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+, but for a specific record delimiter.</p><p>By default, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+will provide one record per line (per <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">'\n'</code>).
+But what if the format to read considers that one record is stored in
+more than one line (yml, custom format, ...)?</p><p>For instance in order to read a yml file, which is a format for which a
+record (a single entity) is spread other several lines, you can modify
+the record delimiter with <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">"---\n"</code>
+instead of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">"\n"</code>.
+Same goes when reading an xml file where a record might be spread over
+several lines or worse the whole xml file is one line.</p><pre><span class="cmt">// Let's say data we want to use with Spark looks like this (one record</span>
+<span class="cmt">// is a customer, but it's spread over several lines):</span>
+&lt;Customers&gt;\n
+&lt;Customer&gt;\n
+&lt;Address&gt;<span class="num">34</span> thingy street, someplace, sometown&lt;/Address&gt;\n
+&lt;/Customer&gt;\n
+&lt;Customer&gt;\n
+&lt;Address&gt;<span class="num">12</span> thingy street, someplace, sometown&lt;/Address&gt;\n
+&lt;/Customer&gt;\n
+&lt;/Customers&gt;
+<span class="cmt">//Then you can use it this way:</span>
+<span class="kw">val</span> computedRecords = sc.textFile(<span class="lit">"my/path/to/customers.xml"</span>, <span class="lit">"<Customer>\n"</span>)
+<span class="kw">val</span> expectedRecords = RDD(
+  &lt;Customers&gt;\n,
+  (
+    &lt;Address&gt;<span class="num">34</span> thingy street, someplace, sometown&lt;/Address&gt;\n +
+    &lt;/Customer&gt;\n
+  ),
+  (
+    &lt;Address&gt;<span class="num">12</span> thingy street, someplace, sometown&lt;/Address&gt;\n +
+    &lt;/Customer&gt;\n +
+    &lt;/Customers&gt;
+  )
+)
+assert(computedRecords == expectedRecords)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the file to read (folder or file, '*' works
+as well).</p></dd><dt class="param">delimiter</dt><dd class="cmt"><p>the specific record delimiter which replaces &quot;\n&quot;</p></dd><dt class="param">maxRecordLength</dt><dd class="cmt"><p>the max length (not sure which unit) of a record
+before considering the record too long to fit into memory.</p></dd><dt>returns</dt><dd class="cmt"><p>the RDD of records</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions#textFileWithFileName" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="textFileWithFileName(path:String):org.apache.spark.rdd.RDD[(String,String)]"></a>
+      <a id="textFileWithFileName(String):RDD[(String,String)]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">textFileWithFileName</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@textFileWithFileName(path:String):org.apache.spark.rdd.RDD[(String,String)]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+, but each record is associated with the file path it comes from.</code></p><div class="fullcomment"><div class="comment cmt"><p>Equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">sparkContext.textFile()</code>
+, but each record is associated with the file path it comes from.</p><p>Produces an <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">RDD[(file_name, line)]</code>
+which provides a way to know from which file a given line comes from.</p><pre><span class="cmt">// Considering this folder:</span>
+<span class="cmt">// folder/file_1.txt whose content is data1\ndata2\ndata3</span>
+<span class="cmt">// folder/file_2.txt whose content is data4\ndata4</span>
+<span class="cmt">// folder/folder_1/file_3.txt whose content is data6\ndata7</span>
+<span class="cmt">// then:</span>
+sc.textFileWithFileName(<span class="lit">"folder"</span>)
+<span class="cmt">// will return:</span>
+RDD(
+  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data1"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data2"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data3"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/file_2.txt"</span>, <span class="lit">"data4"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/file_2.txt"</span>, <span class="lit">"data5"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/folder_1/file_3.txt"</span>, <span class="lit">"data6"</span>),
+  (<span class="lit">"file:/path/on/machine/folder/folder_1/file_3.txt"</span>, <span class="lit">"data7"</span>)
+)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the folder (or structure of folders) to read</p></dd><dt>returns</dt><dd class="cmt"><p>the RDD of records where a record is a tuple containing the path
+of the file the record comes from and the record itself.</p></dd></dl></div>
+    </li><li name="scala.Any#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$SparkContextExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyVal">
+              <h3>Inherited from <span class="extype" name="scala.AnyVal">AnyVal</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$$StringRDDExtensions.html b/docs/com/spark_helper/SparkHelper$$StringRDDExtensions.html
new file mode 100644
index 0000000..bf483ea
--- /dev/null
+++ b/docs/com/spark_helper/SparkHelper$$StringRDDExtensions.html
@@ -0,0 +1,422 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>StringRDDExtensions - com.spark_helper.SparkHelper.StringRDDExtensions</title>
+          <meta name="description" content="StringRDDExtensions - com.spark helper.SparkHelper.StringRDDExtensions" />
+          <meta name="keywords" content="StringRDDExtensions com.spark helper.SparkHelper.StringRDDExtensions" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'com.spark_helper.SparkHelper$$StringRDDExtensions';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="type">
+      <div id="definition">
+        <img alt="Class" src="../../lib/class_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="com">com</a>.<a href="package.html" class="extype" name="com.spark_helper">spark_helper</a>.<a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a></p>
+        <h1>StringRDDExtensions</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">package SparkHelper</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringRDDExtensions</span><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyVal">AnyVal</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="com.spark_helper.SparkHelper.StringRDDExtensions"><span>StringRDDExtensions</span></li><li class="in" name="scala.AnyVal"><span>AnyVal</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        <div id="constructors" class="members">
+              <h3>Instance Constructors</h3>
+              <ol><li name="com.spark_helper.SparkHelper.StringRDDExtensions#&lt;init&gt;" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[String]):com.spark_helper.SparkHelper.StringRDDExtensions"></a>
+      <a id="&lt;init&gt;:StringRDDExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">new</span>
+      </span>
+      <span class="symbol">
+        <span class="name">StringRDDExtensions</span><span class="params">(<span name="rdd">rdd: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>)</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@&lt;init&gt;(rdd:org.apache.spark.rdd.RDD[String]):com.spark_helper.SparkHelper.StringRDDExtensions" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.Any#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@##():Int" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyVal#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <a id="getClass():Class[_&lt;:AnyVal]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="scala.AnyVal">AnyVal</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@getClass():Class[_&lt;:AnyVal]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyVal → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#rdd" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="rdd:org.apache.spark.rdd.RDD[String]"></a>
+      <a id="rdd:RDD[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">rdd</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@rdd:org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsSingleTextFile(path:String,workingFolder:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="saveAsSingleTextFile(String,String,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="workingFolder">workingFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsSingleTextFile(path:String,workingFolder:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
+distributed.</p><p>This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
+performs the storage in a temporary folder instead of directly in the
+final output folder. This way the risks of having corrupted files in the
+real output folder due to cluster interruptions is minimized.</p><pre>rdd.saveAsSingleTextFile(<span class="lit">"/my/file/path.txt"</span>, <span class="lit">"/my/working/folder/path"</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">workingFolder</dt><dd class="cmt"><p>the path where file manipulations will temporarily
+happen.</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsSingleTextFile(path:String,workingFolder:String):Unit"></a>
+      <a id="saveAsSingleTextFile(String,String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="workingFolder">workingFolder: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsSingleTextFile(path:String,workingFolder:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
+distributed.</p><p>This variant of <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">saveAsSingleTextFile</code>
+performs the storage in a temporary folder instead of directly in the
+final output folder. This way the risks of having corrupted files in the
+real output folder due to cluster interruptions is minimized.</p><pre>rdd.saveAsSingleTextFile(<span class="lit">"/my/file/path.txt"</span>, <span class="lit">"/my/working/folder/path"</span>)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">workingFolder</dt><dd class="cmt"><p>the path where file manipulations will temporarily
+happen.</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsSingleTextFile(path:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="saveAsSingleTextFile(String,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsSingleTextFile(path:String,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
+distributed.</p><pre>rdd.saveAsSingleTextFile(<span class="lit">"/my/file/path.txt"</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsSingleTextFile(path:String):Unit"></a>
+      <a id="saveAsSingleTextFile(String):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsSingleTextFile(path:String):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
+distributed.</p><pre>rdd.saveAsSingleTextFile(<span class="lit">"/my/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the path of the produced file</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsTextFileAndCoalesce" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileAndCoalesce(path:String,finalCoalesceLevel:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
+      <a id="saveAsTextFileAndCoalesce(String,Int,Class[_&lt;:CompressionCodec]):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileAndCoalesce</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalesceLevel">finalCoalesceLevel: <span class="extype" name="scala.Int">Int</span></span>, <span name="codec">codec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsTextFileAndCoalesce(path:String,finalCoalesceLevel:Int,codec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves as text file, and decreases the nbr of output partitions.</p><div class="fullcomment"><div class="comment cmt"><p>Saves as text file, and decreases the nbr of output partitions.</p><p>Same as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.saveAsTextFile()</code>
+, but decreases the nbr of partitions in the output folder before doing
+so.</p><p>The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
+, but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
+is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
+would make the processing time explode, whereas this methods keeps the
+processing distributed, save as text file and then only merges the
+result in a lower nbr of partitions.</p><pre>rdd.saveAsTextFileAndCoalesce(<span class="lit">"/produced/folder/path/with/only/30/files"</span>, <span class="num">30</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will finally be stored the RDD but spread
+on only 30 files (where 30 is the value of the finalCoalesceLevel
+parameter).</p></dd><dt class="param">finalCoalesceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
+of this method.</p></dd><dt class="param">codec</dt><dd class="cmt"><p>the type of compression to use (for instance
+classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions#saveAsTextFileAndCoalesce" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="saveAsTextFileAndCoalesce(path:String,finalCoalesceLevel:Int):Unit"></a>
+      <a id="saveAsTextFileAndCoalesce(String,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">saveAsTextFileAndCoalesce</span><span class="params">(<span name="path">path: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalesceLevel">finalCoalesceLevel: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@saveAsTextFileAndCoalesce(path:String,finalCoalesceLevel:Int):Unit" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <p class="shortcomment cmt">Saves as text file, but by decreasing the nbr of partitions of the output.</p><div class="fullcomment"><div class="comment cmt"><p>Saves as text file, but by decreasing the nbr of partitions of the output.</p><p>Same as <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.saveAsTextFile()</code>
+, but decreases the nbr of partitions in the output folder before doing
+so.</p><p>The result is equivalent to <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">rdd.coalesce(x).saveAsTextFile()</code>
+, but if <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">x</code>
+is very low, <code style="background-color:#eff0f1;padding:1px 5px;font-size:12px">coalesce</code>
+would make the processing time explode, whereas this methods keeps the
+processing distributed, save as text file and then only merges the
+result in a lower nbr of partitions.</p><pre>rdd.saveAsTextFileAndCoalesce(<span class="lit">"/produced/folder/path/with/only/30/files"</span>, <span class="num">30</span>)</pre></div><dl class="paramcmts block"><dt class="param">path</dt><dd class="cmt"><p>the folder where will finally be stored the RDD but spread
+on only 30 files (where 30 is the value of the finalCoalesceLevel
+parameter).</p></dd><dt class="param">finalCoalesceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
+of this method.</p></dd></dl></div>
+    </li><li name="scala.Any#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$$StringRDDExtensions@toString():String" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyVal">
+              <h3>Inherited from <span class="extype" name="scala.AnyVal">AnyVal</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/com/spark_helper/SparkHelper$.html b/docs/com/spark_helper/SparkHelper$.html
index f7024b1..25d8011 100644
--- a/docs/com/spark_helper/SparkHelper$.html
+++ b/docs/com/spark_helper/SparkHelper$.html
@@ -51,19 +51,39 @@ <h4 id="signature" class="signature">
       
           <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>A facility to deal with RDD/file manipulations based on the Spark API.</p><p>The goal is to remove the maximum of highly used low-level code from your
 spark job and replace it with methods fully tested whose name is
-self-explanatory/readable.</p><p>A few exemples:</p><pre><span class="cmt">// Same as SparkContext.saveAsTextFile, but the result is a single file:</span>
-SparkHelper.saveAsSingleTextFile(myOutputRDD, <span class="lit">"/my/output/file/path.txt"</span>)
-<span class="cmt">// Same as SparkContext.textFile, but instead of reading one record per</span>
-<span class="cmt">// line, it reads records spread over several lines.</span>
-<span class="cmt">// This way, xml, json, yml or any multi-line record file format can be used</span>
-<span class="cmt">// with Spark:</span>
-SparkHelper.textFileWithDelimiter(<span class="lit">"/my/input/folder/path"</span>, sparkContext, <span class="lit">"---\n"</span>)
-<span class="cmt">// Same as SparkContext.textFile, but instead of returning an RDD of</span>
-<span class="cmt">// records, it returns an RDD of tuples containing both the record and the</span>
-<span class="cmt">// path of the file it comes from:</span>
-SparkHelper.textFileWithFileName(<span class="lit">"folder"</span>, sparkContext)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
+self-explanatory/readable.</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.SparkHelper._
+
+<span class="cmt">// Same as rdd.saveAsTextFile("path"), but the result is a single file (while</span>
+<span class="cmt">// keeping the processing distributed):</span>
+rdd.saveAsSingleTextFile(<span class="lit">"/my/output/file/path.txt"</span>)
+rdd.saveAsSingleTextFile(<span class="lit">"/my/output/file/path.txt"</span>, classOf[BZip2Codec])
+
+<span class="cmt">// Same as sc.textFile("path"), but instead of reading one record per line (by</span>
+<span class="cmt">// splitting the input with \n), it splits the file in records based on a custom</span>
+<span class="cmt">// delimiter. This way, xml, json, yml or any multi-line record file format can</span>
+<span class="cmt">// be used with Spark:</span>
+sc.textFile(<span class="lit">"/my/input/folder/path"</span>, <span class="lit">"---\n"</span>) <span class="cmt">// for a yml file for instance</span>
+
+<span class="cmt">// Equivalent to rdd.flatMap(identity) for RDDs of Seqs or Options:</span>
+rdd.flatten
+
+<span class="cmt">// Equivalent to sc.textFile(), but for each line is tupled with its file path:</span>
+sc.textFileWithFileName(<span class="lit">"/my/input/folder/path"</span>)
+<span class="cmt">// which produces:</span>
+<span class="cmt">// RDD(("folder/file_1.txt", "record1fromfile1"), ("folder/file_1.txt", "record2fromfile1"),</span>
+<span class="cmt">//    ("folder/file_2.txt", "record1fromfile2"), ...)</span>
+
+<span class="cmt">// In the given folder, this generates one file per key in the given key/value</span>
+<span class="cmt">// RDD. Within each file (named from the key) are all values for this key:</span>
+rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>)
+
+<span class="cmt">// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):</span>
+rdd.partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
+
+<span class="cmt">// For when input files contain commas and textFile can't handle it:</span>
+sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/SparkHelper.scala">SparkHelper</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl><div class="toggleContainer block">
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>sc.parallelize[T](elmts: T*) instead of sc.parallelize[T](elmts: Array[T])</p></span></dd></dl><div class="toggleContainer block">
           <span class="toggle">Linear Supertypes</span>
           <div class="superTypes hiddenContent"><span class="extype" name="scala.Serializable">Serializable</span>, <span class="extype" name="java.io.Serializable">Serializable</span>, <span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
         </div></div>
@@ -102,7 +122,112 @@ <h4 id="signature" class="signature">
         <div id="allMembers">
         
 
-        
+        <div id="types" class="types members">
+              <h3>Type Members</h3>
+              <ol><li name="com.spark_helper.SparkHelper.OptionRDDExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="OptionRDDExtensions[T]extendsAnyRef"></a>
+      <a id="OptionRDDExtensions[T]:OptionRDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$OptionRDDExtensions.html"><span class="name">OptionRDDExtensions</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@OptionRDDExtensions[T]extendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.PairRDDExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="PairRDDExtensionsextendsAnyVal"></a>
+      <a id="PairRDDExtensions:PairRDDExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$PairRDDExtensions.html"><span class="name">PairRDDExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@PairRDDExtensionsextendsAnyVal" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.RDDExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="RDDExtensions[T]extendsAnyRef"></a>
+      <a id="RDDExtensions[T]:RDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$RDDExtensions.html"><span class="name">RDDExtensions</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@RDDExtensions[T]extendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.SeqRDDExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SeqRDDExtensions[T]extendsAnyRef"></a>
+      <a id="SeqRDDExtensions[T]:SeqRDDExtensions[T]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$SeqRDDExtensions.html"><span class="name">SeqRDDExtensions</span></a><span class="tparams">[<span name="T">T</span>]</span><span class="result"> extends <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@SeqRDDExtensions[T]extendsAnyRef" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.SparkContextExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SparkContextExtensionsextendsAnyVal"></a>
+      <a id="SparkContextExtensions:SparkContextExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$SparkContextExtensions.html"><span class="name">SparkContextExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@SparkContextExtensionsextendsAnyVal" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="com.spark_helper.SparkHelper.StringRDDExtensions" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="StringRDDExtensionsextendsAnyVal"></a>
+      <a id="StringRDDExtensions:StringRDDExtensions"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">implicit final </span>
+        <span class="kind">class</span>
+      </span>
+      <span class="symbol">
+        <a href="SparkHelper$$StringRDDExtensions.html"><span class="name">StringRDDExtensions</span></a><span class="result"> extends <span class="extype" name="scala.AnyVal">AnyVal</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#com.spark_helper.SparkHelper$@StringRDDExtensionsextendsAnyVal" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
 
         
 
@@ -199,62 +324,6 @@ <h4 class="signature">
     </span>)</span>
               
         </dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#decreaseCoalescence" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalescenceLevel:Int,sparkContext:org.apache.spark.SparkContext,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
-      <a id="decreaseCoalescence(String,String,Int,SparkContext,Class[_&lt;:CompressionCodec]):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">decreaseCoalescence</span><span class="params">(<span name="highCoalescenceLevelFolder">highCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lowerCoalescenceLevelFolder">lowerCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalescenceLevel">finalCoalescenceLevel: <span class="extype" name="scala.Int">Int</span></span>, <span name="sparkContext">sparkContext: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>, <span name="compressionCodec">compressionCodec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalescenceLevel:Int,sparkContext:org.apache.spark.SparkContext,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Decreases the nbr of partitions of a folder.</p><div class="fullcomment"><div class="comment cmt"><p>Decreases the nbr of partitions of a folder.</p><p>This is often handy when the last step of your job needs to run on
-thousands of files, but you want to store your final output on let's say
-only 300 files.</p><p>It's like a FileUtil.copyMerge, but the merging produces more than one
-file.</p><p>Be aware that this methods deletes the provided input folder.</p><pre>SparkHelper.decreaseCoalescence(
-  <span class="lit">"/folder/path/with/2000/files"</span>,
-  <span class="lit">"/produced/folder/path/with/only/300/files"</span>,
-  <span class="num">300</span>,
-  sparkContext,
-  classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">highCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which contains 10000 files</p></dd><dt class="param">lowerCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which will contain the same
-data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-is the finalCoalescenceLevel parameter).</p></dd><dt class="param">finalCoalescenceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
-of this method.</p></dd><dt class="param">sparkContext</dt><dd class="cmt"><p>the SparkContext</p></dd><dt class="param">compressionCodec</dt><dd class="cmt"><p>the type of compression to use (for instance
-classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#decreaseCoalescence" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalescenceLevel:Int,sparkContext:org.apache.spark.SparkContext):Unit"></a>
-      <a id="decreaseCoalescence(String,String,Int,SparkContext):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">decreaseCoalescence</span><span class="params">(<span name="highCoalescenceLevelFolder">highCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="lowerCoalescenceLevelFolder">lowerCoalescenceLevelFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalescenceLevel">finalCoalescenceLevel: <span class="extype" name="scala.Int">Int</span></span>, <span name="sparkContext">sparkContext: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@decreaseCoalescence(highCoalescenceLevelFolder:String,lowerCoalescenceLevelFolder:String,finalCoalescenceLevel:Int,sparkContext:org.apache.spark.SparkContext):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Decreases the nbr of partitions of a folder.</p><div class="fullcomment"><div class="comment cmt"><p>Decreases the nbr of partitions of a folder.</p><p>This is often handy when the last step of your job needs to run on
-thousands of files, but you want to store your final output on let's say
-only 300 files.</p><p>It's like a FileUtil.copyMerge, but the merging produces more than one
-file.</p><p>Be aware that this methods deletes the provided input folder.</p><pre>SparkHelper.decreaseCoalescence(
-  <span class="lit">"/folder/path/with/2000/files"</span>,
-  <span class="lit">"/produced/folder/path/with/only/300/files"</span>,
-  <span class="num">300</span>,
-  sparkContext)</pre></div><dl class="paramcmts block"><dt class="param">highCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which contains 10000 files</p></dd><dt class="param">lowerCoalescenceLevelFolder</dt><dd class="cmt"><p>the folder which will contain the same
-data as highCoalescenceLevelFolder but spread on only 300 files (where 300
-is the finalCoalescenceLevel parameter).</p></dd><dt class="param">finalCoalescenceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
-of this method.</p></dd><dt class="param">sparkContext</dt><dd class="cmt"><p>the SparkContext</p></dd></dl></div>
     </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="eq(x$1:AnyRef):Boolean"></a>
       <a id="eq(AnyRef):Boolean"></a>
@@ -414,193 +483,6 @@ <h4 class="signature">
       </a>
     </span>
       <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,workingFolder:String,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
-      <a id="saveAsSingleTextFile(RDD[String],String,String,Class[_&lt;:CompressionCodec]):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFile">outputFile: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="workingFolder">workingFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="compressionCodec">compressionCodec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,workingFolder:String,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
-parallelized.</p><p>This variant of saveAsSingleTextFile performs the storage in a temporary
-folder instead of directly in the final output folder. This way the risks
-of having corrupted files in the real output folder due to cluster
-interruptions is minimized.</p><pre>SparkHelper.saveAsSingleTextFile(
-  myRddToStore,
-  <span class="lit">"/my/file/path.txt"</span>,
-  <span class="lit">"/my/working/folder/path"</span>,
-  classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD of strings to store in one file</p></dd><dt class="param">outputFile</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">workingFolder</dt><dd class="cmt"><p>the path where file manipulations will temporarily
-happen.</p></dd><dt class="param">compressionCodec</dt><dd class="cmt"><p>the type of compression to use (for instance
-classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,workingFolder:String):Unit"></a>
-      <a id="saveAsSingleTextFile(RDD[String],String,String):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFile">outputFile: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="workingFolder">workingFolder: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,workingFolder:String):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
-parallelized.</p><p>This variant of saveAsSingleTextFile performs the storage in a temporary
-folder instead of directly in the final output folder. This way the
-risks of having corrupted files in the real output folder due to cluster
-interruptions is minimized.</p><pre>SparkHelper.saveAsSingleTextFile(
-  myRddToStore, <span class="lit">"/my/file/path.txt"</span>, <span class="lit">"/my/working/folder/path"</span>)</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD of strings to store in one file</p></dd><dt class="param">outputFile</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">workingFolder</dt><dd class="cmt"><p>the path where file manipulations will temporarily
-happen.</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
-      <a id="saveAsSingleTextFile(RDD[String],String,Class[_&lt;:CompressionCodec]):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFile">outputFile: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="compressionCodec">compressionCodec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
-parallelized.</p><pre>SparkHelper.saveAsSingleTextFile(
-  myRddToStore, <span class="lit">"/my/file/path.txt"</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD of strings to store in one file</p></dd><dt class="param">outputFile</dt><dd class="cmt"><p>the path of the produced file</p></dd><dt class="param">compressionCodec</dt><dd class="cmt"><p>the type of compression to use (for instance
-classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsSingleTextFile" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String):Unit"></a>
-      <a id="saveAsSingleTextFile(RDD[String],String):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsSingleTextFile</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFile">outputFile: <span class="extype" name="scala.Predef.String">String</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsSingleTextFile(outputRDD:org.apache.spark.rdd.RDD[String],outputFile:String):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves an RDD in exactly one file.</p><div class="fullcomment"><div class="comment cmt"><p>Saves an RDD in exactly one file.</p><p>Allows one to save an RDD in one file, while keeping the processing
-parallelized.</p><pre>SparkHelper.saveAsSingleTextFile(myRddToStore, <span class="lit">"/my/file/path.txt"</span>)</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD of strings to store in one file</p></dd><dt class="param">outputFile</dt><dd class="cmt"><p>the path of the produced file</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsTextFileAndCoalesce" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsTextFileAndCoalesce(outputRDD:org.apache.spark.rdd.RDD[String],outputFolder:String,finalCoalescenceLevel:Int,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
-      <a id="saveAsTextFileAndCoalesce(RDD[String],String,Int,Class[_&lt;:CompressionCodec]):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsTextFileAndCoalesce</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFolder">outputFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalescenceLevel">finalCoalescenceLevel: <span class="extype" name="scala.Int">Int</span></span>, <span name="compressionCodec">compressionCodec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsTextFileAndCoalesce(outputRDD:org.apache.spark.rdd.RDD[String],outputFolder:String,finalCoalescenceLevel:Int,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves as text file, but by decreasing the nbr of partitions of the output.</p><div class="fullcomment"><div class="comment cmt"><p>Saves as text file, but by decreasing the nbr of partitions of the output.</p><p>Same as decreaseCoalescence, but the storage of the RDD in an intermediate
-folder is included.</p><p>This still makes the processing parallelized, but the output is coalesced.</p><pre>SparkHelper.saveAsTextFileAndCoalesce(
-  myRddToStore,
-  <span class="lit">"/produced/folder/path/with/only/300/files"</span>,
-  <span class="num">300</span>,
-  classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD to store, processed for instance on 10000 tasks
-(which would thus be stored as 10000 files).</p></dd><dt class="param">outputFolder</dt><dd class="cmt"><p>the folder where will finally be stored the RDD but
-spread on only 300 files (where 300 is the value of the
-finalCoalescenceLevel parameter).</p></dd><dt class="param">finalCoalescenceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
-of this method.</p></dd><dt class="param">compressionCodec</dt><dd class="cmt"><p>the type of compression to use (for instance
-classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsTextFileAndCoalesce" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsTextFileAndCoalesce(outputRDD:org.apache.spark.rdd.RDD[String],outputFolder:String,finalCoalescenceLevel:Int):Unit"></a>
-      <a id="saveAsTextFileAndCoalesce(RDD[String],String,Int):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsTextFileAndCoalesce</span><span class="params">(<span name="outputRDD">outputRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="outputFolder">outputFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="finalCoalescenceLevel">finalCoalescenceLevel: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsTextFileAndCoalesce(outputRDD:org.apache.spark.rdd.RDD[String],outputFolder:String,finalCoalescenceLevel:Int):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves as text file, but by decreasing the nbr of partitions of the output.</p><div class="fullcomment"><div class="comment cmt"><p>Saves as text file, but by decreasing the nbr of partitions of the output.</p><p>Same as decreaseCoalescence, but the storage of the RDD in an intermediate
-folder is included.</p><p>This still makes the processing parallelized, but the output is coalesced.</p><pre>SparkHelper.saveAsTextFileAndCoalesce(
-  myRddToStore, <span class="lit">"/produced/folder/path/with/only/300/files"</span>, <span class="num">300</span>)</pre></div><dl class="paramcmts block"><dt class="param">outputRDD</dt><dd class="cmt"><p>the RDD to store, processed for instance on 10000 tasks
-(which would thus be stored as 10000 files).</p></dd><dt class="param">outputFolder</dt><dd class="cmt"><p>the folder where will finally be stored the RDD but
-spread on only 300 files (where 300 is the value of the
-finalCoalescenceLevel parameter).</p></dd><dt class="param">finalCoalescenceLevel</dt><dd class="cmt"><p>the nbr of files within the folder at the end
-of this method.</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsTextFileByKey(keyValueRDD:org.apache.spark.rdd.RDD[(String,String)],outputFolder:String,keyNbr:Int,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit"></a>
-      <a id="saveAsTextFileByKey(RDD[(String,String)],String,Int,Class[_&lt;:CompressionCodec]):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="keyValueRDD">keyValueRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>, <span name="outputFolder">outputFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="keyNbr">keyNbr: <span class="extype" name="scala.Int">Int</span></span>, <span name="compressionCodec">compressionCodec: <span class="extype" name="scala.Predef.Class">Class</span>[_ &lt;: <span class="extype" name="org.apache.hadoop.io.compress.CompressionCodec">CompressionCodec</span>]</span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsTextFileByKey(keyValueRDD:org.apache.spark.rdd.RDD[(String,String)],outputFolder:String,keyNbr:Int,compressionCodec:Class[_&lt;:org.apache.hadoop.io.compress.CompressionCodec]):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided outputFolder, will be one file per key in your
-keyValueRDD. And within a file for a given key are only values for this
-key.</p><p>You need to know the nbr of keys beforehand (in general you use this to
-split your dataset in subsets, or to output one file per client, so you
-know how many keys you have). So you need to put as keyNbr the exact nbr
-of keys you'll have.</p><p>This is not scalable. This shouldn't be considered for any data flow with
-normal or big volumes.</p><pre>SparkHelper.saveAsTextFileByKey(
-  myKeyValueRddToStore, <span class="lit">"/my/output/folder/path"</span>, <span class="num">12</span>, classOf[BZip2Codec])</pre></div><dl class="paramcmts block"><dt class="param">keyValueRDD</dt><dd class="cmt"><p>the key/value RDD</p></dd><dt class="param">outputFolder</dt><dd class="cmt"><p>the foldder where will be storrred key files</p></dd><dt class="param">keyNbr</dt><dd class="cmt"><p>the nbr of expected keys (which is the nbr of outputed files)</p></dd><dt class="param">compressionCodec</dt><dd class="cmt"><p>the type of compression to use (for instance
-classOf[BZip2Codec] or classOf[GzipCodec]))</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#saveAsTextFileByKey" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="saveAsTextFileByKey(keyValueRDD:org.apache.spark.rdd.RDD[(String,String)],outputFolder:String,keyNbr:Int):Unit"></a>
-      <a id="saveAsTextFileByKey(RDD[(String,String)],String,Int):Unit"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">saveAsTextFileByKey</span><span class="params">(<span name="keyValueRDD">keyValueRDD: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>, <span name="outputFolder">outputFolder: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="keyNbr">keyNbr: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@saveAsTextFileByKey(keyValueRDD:org.apache.spark.rdd.RDD[(String,String)],outputFolder:String,keyNbr:Int):Unit" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Saves and repartitions a key/value RDD on files whose name is the key.</p><div class="fullcomment"><div class="comment cmt"><p>Saves and repartitions a key/value RDD on files whose name is the key.</p><p>Within the provided outputFolder, will be one file per key in your
-keyValueRDD. And within a file for a given key are only values for this
-key.</p><p>You need to know the nbr of keys beforehand (in general you use this to
-split your dataset in subsets, or to output one file per client, so you
-know how many keys you have). So you need to put as keyNbr the exact nbr
-of keys you'll have.</p><p>This is not scalable. This shouldn't be considered for any data flow with
-normal or big volumes.</p><pre>SparkHelper.saveAsTextFileByKey(
-  myKeyValueRddToStore, <span class="lit">"/my/output/folder/path"</span>, <span class="num">12</span>)</pre></div><dl class="paramcmts block"><dt class="param">keyValueRDD</dt><dd class="cmt"><p>the key/value RDD</p></dd><dt class="param">outputFolder</dt><dd class="cmt"><p>the foldder where will be storrred key files</p></dd><dt class="param">keyNbr</dt><dd class="cmt"><p>the nbr of expected keys (which is the nbr of outputed files)</p></dd></dl></div>
     </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
       <a id="synchronized[T0](⇒T0):T0"></a>
@@ -618,93 +500,6 @@ <h4 class="signature">
       </a>
     </span>
       <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#textFileWithDelimiter" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="textFileWithDelimiter(hdfsPath:String,sparkContext:org.apache.spark.SparkContext,delimiter:String,maxRecordLength:String):org.apache.spark.rdd.RDD[String]"></a>
-      <a id="textFileWithDelimiter(String,SparkContext,String,String):RDD[String]"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">textFileWithDelimiter</span><span class="params">(<span name="hdfsPath">hdfsPath: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="sparkContext">sparkContext: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>, <span name="delimiter">delimiter: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="maxRecordLength">maxRecordLength: <span class="extype" name="scala.Predef.String">String</span> = <span class="symbol">&quot;1000000&quot;</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@textFileWithDelimiter(hdfsPath:String,sparkContext:org.apache.spark.SparkContext,delimiter:String,maxRecordLength:String):org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Equivalent to sparkContext.textFile(), but for a specific record delimiter.</p><div class="fullcomment"><div class="comment cmt"><p>Equivalent to sparkContext.textFile(), but for a specific record delimiter.</p><p>By default, sparkContext.textFile() will provide one record per line. But
-what if the format you want to read considers that one record (one entity)
-is stored in more than one line (yml, xml, ...)?</p><p>For instance in order to read a yml file, which is a format for which a
-record (a single entity) is spread other several lines, you can modify the
-record delimiter with &quot;---\n&quot; instead of &quot;\n&quot;. Same goes when reading an
-xml file where a record might be spread over several lines or worse the
-whole xml file is one line.</p><pre><span class="cmt">// Let's say data we want to use with Spark looks like this (one record is</span>
-<span class="cmt">// a customer, but it's spread over several lines):</span>
-&lt;Customers&gt;\n
-&lt;Customer&gt;\n
-&lt;Address&gt;<span class="num">34</span> thingy street, someplace, sometown&lt;/Address&gt;\n
-&lt;/Customer&gt;\n
-&lt;Customer&gt;\n
-&lt;Address&gt;<span class="num">12</span> thingy street, someplace, sometown&lt;/Address&gt;\n
-&lt;/Customer&gt;\n
-&lt;/Customers&gt;
-<span class="cmt">//Then you can use it this way:</span>
-<span class="kw">val</span> computedRecords = SparkHelper.textFileWithDelimiter(
-  <span class="lit">"my/path/to/customers.xml"</span>, sparkContext, &lt;Customer&gt;\n
-).collect()
-<span class="kw">val</span> expectedRecords = <span class="std">Array</span>(
-  &lt;Customers&gt;\n,
-  (
-    &lt;Address&gt;<span class="num">34</span> thingy street, someplace, sometown&lt;/Address&gt;\n +
-    &lt;/Customer&gt;\n
-  ),
-  (
-    &lt;Address&gt;<span class="num">12</span> thingy street, someplace, sometown&lt;/Address&gt;\n +
-    &lt;/Customer&gt;\n +
-    &lt;/Customers&gt;
-  )
-)
-assert(computedRecords == expectedRecords)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the file to read (folder or file, '*' works as
-well).</p></dd><dt class="param">sparkContext</dt><dd class="cmt"><p>the SparkContext</p></dd><dt class="param">delimiter</dt><dd class="cmt"><p>the specific record delimiter which replaces &quot;\n&quot;</p></dd><dt class="param">maxRecordLength</dt><dd class="cmt"><p>the max length (not sure which unit) of a record
-before considering the record too long to fit into memory.</p></dd><dt>returns</dt><dd class="cmt"><p>the RDD of records</p></dd></dl></div>
-    </li><li name="com.spark_helper.SparkHelper#textFileWithFileName" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
-      <a id="textFileWithFileName(hdfsPath:String,sparkContext:org.apache.spark.SparkContext):org.apache.spark.rdd.RDD[(String,String)]"></a>
-      <a id="textFileWithFileName(String,SparkContext):RDD[(String,String)]"></a>
-      <h4 class="signature">
-      <span class="modifier_kind">
-        <span class="modifier"></span>
-        <span class="kind">def</span>
-      </span>
-      <span class="symbol">
-        <span class="name">textFileWithFileName</span><span class="params">(<span name="hdfsPath">hdfsPath: <span class="extype" name="scala.Predef.String">String</span></span>, <span name="sparkContext">sparkContext: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[(<span class="extype" name="scala.Predef.String">String</span>, <span class="extype" name="scala.Predef.String">String</span>)]</span>
-      </span>
-      </h4><span class="permalink">
-      <a href="../../index.html#com.spark_helper.SparkHelper$@textFileWithFileName(hdfsPath:String,sparkContext:org.apache.spark.SparkContext):org.apache.spark.rdd.RDD[(String,String)]" title="Permalink" target="_top">
-        <img src="../../lib/permalink.png" alt="Permalink" />
-      </a>
-    </span>
-      <p class="shortcomment cmt">Equivalent to sparkContext.textFile(), but for each line is associated
-with its file path.</p><div class="fullcomment"><div class="comment cmt"><p>Equivalent to sparkContext.textFile(), but for each line is associated
-with its file path.</p><p>Produces a RDD[(file_name, line)] which provides a way to know from which
-file a given line comes from.</p><pre><span class="cmt">// Considering this folder:</span>
-<span class="cmt">// folder/file_1.txt whose content is data1\ndata2\ndata3</span>
-<span class="cmt">// folder/file_2.txt whose content is data4\ndata4</span>
-<span class="cmt">// folder/folder_1/file_3.txt whose content is data6\ndata7</span>
-<span class="cmt">// then:</span>
-SparkHelper.textFileWithFileName(<span class="lit">"folder"</span>, sparkContext)
-<span class="cmt">// will return:</span>
-RDD(
-  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data1"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data2"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/file_1.txt"</span>, <span class="lit">"data3"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/file_2.txt"</span>, <span class="lit">"data4"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/file_2.txt"</span>, <span class="lit">"data5"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/folder_1/file_3.txt"</span>, <span class="lit">"data6"</span>),
-  (<span class="lit">"file:/path/on/machine/folder/folder_1/file_3.txt"</span>, <span class="lit">"data7"</span>)
-)</pre></div><dl class="paramcmts block"><dt class="param">hdfsPath</dt><dd class="cmt"><p>the path of the folder (or structure of folders) to read</p></dd><dt class="param">sparkContext</dt><dd class="cmt"><p>the SparkContext</p></dd><dt>returns</dt><dd class="cmt"><p>the RDD of records where a record is a tuple containing the path
-of the file the record comes from and the record itself.</p></dd></dl></div>
     </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="toString():String"></a>
       <a id="toString():String"></a>
diff --git a/docs/com/spark_helper/monitoring/Test.html b/docs/com/spark_helper/monitoring/Test.html
index f1cd8b7..4246185 100644
--- a/docs/com/spark_helper/monitoring/Test.html
+++ b/docs/com/spark_helper/monitoring/Test.html
@@ -50,7 +50,7 @@ <h4 id="signature" class="signature">
       </h4>
       
           <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>A class which represents a KPI to validate.</p><p>This is intended to be used as parameter of Monitor.updateByKpiValidation
-and Monitor.updateByKpisValidation methods.</p><p>Some exemples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
+and Monitor.updateByKpisValidation methods.</p><p>Some examples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
 Test(<span class="lit">"pctOfSomethingElse"</span>, <span class="num">0.27</span>d, SUPERIOR_THAN, <span class="num">0.3</span>d, PCT)
 Test(<span class="lit">"someNbr"</span>, <span class="num">1235</span>d, EQUAL_TO, <span class="num">1235</span>d, NBR)</pre></div><dl class="paramcmts block"><dt class="param">description</dt><dd class="cmt"><p>the name/description of the KPI which will appear on the
 validation report.</p></dd><dt class="param">kpiValue</dt><dd class="cmt"><p>the value for this KPI</p></dd><dt class="param">thresholdType</dt><dd class="cmt"><p>the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or
@@ -110,7 +110,7 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">Creates a Test object.</p><div class="fullcomment"><div class="comment cmt"><p>Creates a Test object.
-Some exemples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
+Some examples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
 Test(<span class="lit">"pctOfSomethingElse"</span>, <span class="num">0.27</span>d, SUPERIOR_THAN, <span class="num">0.3</span>d, PCT)
 Test(<span class="lit">"someNbr"</span>, <span class="num">1235</span>d, EQUAL_TO, <span class="num">1235</span>d, NBR)</pre></div><dl class="paramcmts block"><dt class="param">description</dt><dd class="cmt"><p>the name/description of the KPI which will appear on the
 validation report.</p></dd><dt class="param">kpiValue</dt><dd class="cmt"><p>the value for this KPI</p></dd><dt class="param">thresholdType</dt><dd class="cmt"><p>the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or
diff --git a/docs/com/spark_helper/monitoring/ThresholdType.html b/docs/com/spark_helper/monitoring/ThresholdType.html
index b0c6eef..61447b9 100644
--- a/docs/com/spark_helper/monitoring/ThresholdType.html
+++ b/docs/com/spark_helper/monitoring/ThresholdType.html
@@ -49,7 +49,7 @@ <h4 id="signature" class="signature">
       </span>
       </h4>
       
-          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>An enumeration which represents the type of threshol to use (EQUAL_TO,
+          <div id="comment" class="fullcommenttop"><div class="comment cmt"><p>An enumeration which represents the type of threshold to use (EQUAL_TO,
 SUPERIOR_THAN or INFERIOR_THAN)</p></div><div class="toggleContainer block">
           <span class="toggle">Linear Supertypes</span>
           <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
diff --git a/docs/com/spark_helper/monitoring/package.html b/docs/com/spark_helper/monitoring/package.html
index f89cf5e..c7c36f9 100644
--- a/docs/com/spark_helper/monitoring/package.html
+++ b/docs/com/spark_helper/monitoring/package.html
@@ -100,7 +100,7 @@ <h4 class="signature">
       </a>
     </span>
       <p class="shortcomment cmt">A class which represents a KPI to validate.</p><div class="fullcomment"><div class="comment cmt"><p>A class which represents a KPI to validate.</p><p>This is intended to be used as parameter of Monitor.updateByKpiValidation
-and Monitor.updateByKpisValidation methods.</p><p>Some exemples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
+and Monitor.updateByKpisValidation methods.</p><p>Some examples of Test objects:</p><pre>Test(<span class="lit">"pctOfWhatever"</span>, <span class="num">0.06</span>d, INFERIOR_THAN, <span class="num">0.1</span>d, PCT)
 Test(<span class="lit">"pctOfSomethingElse"</span>, <span class="num">0.27</span>d, SUPERIOR_THAN, <span class="num">0.3</span>d, PCT)
 Test(<span class="lit">"someNbr"</span>, <span class="num">1235</span>d, EQUAL_TO, <span class="num">1235</span>d, NBR)</pre></div><dl class="paramcmts block"><dt class="param">description</dt><dd class="cmt"><p>the name/description of the KPI which will appear on the
 validation report.</p></dd><dt class="param">kpiValue</dt><dd class="cmt"><p>the value for this KPI</p></dd><dt class="param">thresholdType</dt><dd class="cmt"><p>the type of threshold (SUPERIOR_THAN, INFERIOR_THAN or
@@ -121,7 +121,7 @@ <h4 class="signature">
         <img src="../../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">An enumeration which represents the type of threshol to use (EQUAL_TO,
+      <p class="shortcomment cmt">An enumeration which represents the type of threshold to use (EQUAL_TO,
 SUPERIOR_THAN or INFERIOR_THAN)</p>
     </li></ol>
             </div>
diff --git a/docs/com/spark_helper/package.html b/docs/com/spark_helper/package.html
index 0912242..60ce9ba 100644
--- a/docs/com/spark_helper/package.html
+++ b/docs/com/spark_helper/package.html
@@ -90,14 +90,31 @@ <h4 class="signature">
 <a href="http://www.joda.org/joda-time/apidocs/">joda-time</a>).</a></p><div class="fullcomment"><div class="comment cmt"><p>A facility which deals with usual date needs (wrapper around
 <a href="http://www.joda.org/joda-time/apidocs/">joda-time</a>).</p><p>The goal is to remove the maximum of highly used low-level code from your
 spark job and replace it with methods fully tested whose name is
-self-explanatory/readable.</p><p>A few exemples:</p><pre>assert(DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) == <span class="std">List</span>(<span class="lit">"20161230"</span>, <span class="lit">"20161231"</span>, <span class="lit">"20170101"</span>))
-assert(DateHelper.today() == <span class="lit">"20170310"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.yesterday() == <span class="lit">"20170309"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.reformatDate(<span class="lit">"20170327"</span>, <span class="lit">"yyyyMMdd"</span>, <span class="lit">"yyMMdd"</span>) == <span class="lit">"170327"</span>)
-assert(DateHelper.now(<span class="lit">"HH:mm"</span>) == <span class="lit">"10:24"</span>)
-assert(DateHelper.currentTimestamp() == <span class="lit">"1493105229736"</span>)
-assert(DateHelper.nDaysBefore(<span class="num">3</span>) == <span class="lit">"20170307"</span>) <span class="cmt">// If today's "20170310"</span>
-assert(DateHelper.nDaysAfterDate(<span class="num">3</span>, <span class="lit">"20170307"</span>) == <span class="lit">"20170310"</span>)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
+self-explanatory/readable.</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.DateHelper
+
+DateHelper.daysBetween(<span class="lit">"20161230"</span>, <span class="lit">"20170101"</span>) <span class="cmt">// List("20161230", "20161231", "20170101")</span>
+DateHelper.today <span class="cmt">// "20170310"</span>
+DateHelper.yesterday <span class="cmt">// "20170309"</span>
+DateHelper.reformatDate(<span class="lit">"20170327"</span>, <span class="lit">"yyyyMMdd"</span>, <span class="lit">"yyMMdd"</span>) <span class="cmt">// "170327"</span>
+DateHelper.now(<span class="lit">"HH:mm"</span>) <span class="cmt">// "10:24"</span>
+DateHelper.currentTimestamp <span class="cmt">// "1493105229736"</span>
+DateHelper.nDaysBefore(<span class="num">3</span>) <span class="cmt">// "20170307"</span>
+DateHelper.nDaysAfterDate(<span class="num">3</span>, <span class="lit">"20170307"</span>) <span class="cmt">// "20170310"</span>
+DateHelper.nextDay(<span class="lit">"20170310"</span>) <span class="cmt">// "20170311"</span>
+DateHelper.nbrOfDaysSince(<span class="lit">"20170302"</span>) <span class="cmt">// 8</span>
+DateHelper.nbrOfDaysBetween(<span class="lit">"20170327"</span>, <span class="lit">"20170401"</span>) <span class="cmt">// 5</span>
+DateHelper.dayOfWeek(<span class="lit">"20160614"</span>) <span class="cmt">// 2</span>
+
+<span class="kw">import</span> com.spark_helper.DateHelper._
+
+<span class="num">2.</span>daysAgo <span class="cmt">// "20170308"</span>
+<span class="lit">"20161230"</span> to <span class="lit">"20170101"</span> <span class="cmt">// List("20161230", "20161231", "20170101")</span>
+<span class="num">3.</span>daysBefore(<span class="lit">"20170310"</span>) <span class="cmt">// "20170307"</span>
+<span class="num">5.</span>daysAfter <span class="cmt">// "20170315"</span>
+<span class="num">4.</span>daysAfter(<span class="lit">"20170310"</span>) <span class="cmt">// "20170314"</span>
+<span class="lit">"20170302"</span>.isCompliantWith(<span class="lit">"yyyyMMdd"</span>)
+<span class="lit">"20170310"</span>.nextDay <span class="cmt">// "20170311"</span>
+<span class="lit">"20170310"</span>.previousDay <span class="cmt">// "20170309"</span></pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/DateHelper.scala">DateHelper</a>
 </p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl></div>
     </li><li name="com.spark_helper.HdfsHelper" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
@@ -122,38 +139,39 @@ <h4 class="signature">
 spark job and replace it with methods fully tested whose name is
 self-explanatory/readable.</p><p>For instance, one don't want to remove a file from hdfs using 3 lines of
 code and thus could instead just use
-HdfsHelper.deleteFile(&quot;my/hdfs/file/path.csv&quot;).</p><p>A few exemples:</p><pre><span class="kw">import</span> com.spark_helper.HdfsHelper
+HdfsHelper.deleteFile(&quot;my/hdfs/file/path.csv&quot;).</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.HdfsHelper
 
 <span class="cmt">// A bunch of methods wrapping the FileSystem API, such as:</span>
-HdfsHelper.fileExists(<span class="lit">"my/hdfs/file/path.txt"</span>)
-assert(HdfsHelper.listFileNamesInFolder(<span class="lit">"my/folder/path"</span>) == <span class="std">List</span>(<span class="lit">"file_name_1.txt"</span>, <span class="lit">"file_name_2.csv"</span>))
-assert(HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="lit">"20170306"</span>)
-assert(HdfsHelper.nbrOfDaysSinceFileWasLastModified(<span class="lit">"my/hdfs/file/path.txt"</span>) == <span class="num">3</span>)
-HdfsHelper.deleteFile(<span class="lit">"my/hdfs/file/path.csv"</span>)
-HdfsHelper.moveFolder(<span class="lit">"my/hdfs/folder"</span>)
+HdfsHelper.fileExists(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// HdfsHelper.folderExists("my/hdfs/folder")</span>
+HdfsHelper.listFileNamesInFolder(<span class="lit">"my/folder/path"</span>) <span class="cmt">// List("file_name_1.txt", "file_name_2.csv")</span>
+HdfsHelper.fileModificationDate(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// "20170306"</span>
+HdfsHelper.nbrOfDaysSinceFileWasLastModified(<span class="lit">"my/hdfs/file/path.txt"</span>) <span class="cmt">// 3</span>
+HdfsHelper.deleteFile(<span class="lit">"my/hdfs/file/path.csv"</span>) <span class="cmt">// HdfsHelper.deleteFolder("my/hdfs/folder")</span>
+HdfsHelper.moveFolder(<span class="lit">"old/path"</span>, <span class="lit">"new/path"</span>) <span class="cmt">// HdfsHelper.moveFile("old/path.txt", "new/path.txt")</span>
+HdfsHelper.createEmptyHdfsFile(<span class="lit">"/some/hdfs/file/path.token"</span>) <span class="cmt">// HdfsHelper.createFolder("my/hdfs/folder")</span>
+
+<span class="cmt">// File content helpers:</span>
 HdfsHelper.compressFile(<span class="lit">"hdfs/path/to/uncompressed_file.txt"</span>, classOf[GzipCodec])
 HdfsHelper.appendHeader(<span class="lit">"my/hdfs/file/path.csv"</span>, <span class="lit">"colum0,column1"</span>)
 
 <span class="cmt">// Some Xml/Typesafe helpers for hadoop as well:</span>
-HdfsHelper.isHdfsXmlCompliantWithXsd(
-  <span class="lit">"my/hdfs/file/path.xml"</span>, getClass.getResource(<span class="lit">"/some_xml.xsd"</span>))
+HdfsHelper.isHdfsXmlCompliantWithXsd(<span class="lit">"my/hdfs/file/path.xml"</span>, getClass.getResource(<span class="lit">"/some_xml.xsd"</span>))
 HdfsHelper.loadXmlFileFromHdfs(<span class="lit">"my/hdfs/file/path.xml"</span>)
 
-<span class="cmt">// Very handy to load a config (typesafe format) stored on hdfs at the</span>
-<span class="cmt">// begining of a spark job:</span>
+<span class="cmt">// Very handy to load a config (typesafe format) stored on hdfs at the beginning of a spark job:</span>
 HdfsHelper.loadTypesafeConfigFromHdfs(<span class="lit">"my/hdfs/file/path.conf"</span>): Config
 
-<span class="cmt">// In order to write small amount of data in a file on hdfs without the</span>
-<span class="cmt">// whole spark stack:</span>
-HdfsHelper.writeToHdfsFile(
-  <span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>),
-  <span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="cmt">// In order to write small amount of data in a file on hdfs without the whole spark stack:</span>
+HdfsHelper.writeToHdfsFile(<span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>), <span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="cmt">// or:</span>
+<span class="kw">import</span> com.spark_helper.HdfsHelper._
+<span class="std">Array</span>(<span class="lit">"some"</span>, <span class="lit">"relatively small"</span>, <span class="lit">"text"</span>).writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)
+<span class="lit">"hello world"</span>.writeToHdfs(<span class="lit">"/some/hdfs/file/path.txt"</span>)
 
-<span class="cmt">// Deletes all files/folders in "hdfs/path/to/folder" for which the</span>
-<span class="cmt">// timestamp is older than 10 days:</span>
+<span class="cmt">// Deletes all files/folders in "hdfs/path/to/folder" for which the timestamp is older than 10 days:</span>
 HdfsHelper.purgeFolder(<span class="lit">"hdfs/path/to/folder"</span>, <span class="num">10</span>)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/HdfsHelper.scala">HdfsHelper</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl></div>
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>Create a touch method</p></span></dd></dl></div>
     </li><li name="com.spark_helper.Monitor" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="Monitor"></a>
       <a id="Monitor:Monitor"></a>
@@ -170,11 +188,11 @@ <h4 class="signature">
         <img src="../../lib/permalink.png" alt="Permalink" />
       </a>
     </span>
-      <p class="shortcomment cmt">A logger dedicated to Spak jobs.</p><div class="fullcomment"><div class="comment cmt"><p>A logger dedicated to Spak jobs.</p><p>It's a simple logger/report which contains a report that one can update from
+      <p class="shortcomment cmt">A logger dedicated to Spark jobs.</p><div class="fullcomment"><div class="comment cmt"><p>A logger dedicated to Spark jobs.</p><p>It's a simple logger/report which contains a report that one can update from
 the driver and a success state. The idea is to persist job executions logs
-and errors (and forget about grepping unreadable yarn logs).</p><p>It's designed for perdiodic spark jobs (handles storage and purge of logs)
+and errors (and forget about grepping unreadable yarn logs).</p><p>It's designed for periodic spark jobs (handles storage and purge of logs)
 and provides a way to handle kpis validation.</p><p>Logs are stored on the go which means one can have a direct real time access
-of the job logs/status and it's current state (which can overwise be a pain
+of the job logs/status and it's current state (which can otherwise be a pain
 if it means going through yarn logs, or even for certain production
 environments going through additional layers of software logs to get to yarn
 logs).</p><p>One of the issues this logger aims at tackling is the handling of exceptions
@@ -183,9 +201,9 @@ <h4 class="signature">
 want to perform a few actions before letting the job crash. The idea is thus
 to surround (driver side) a Spark pipeline within a try catch and redirect
 the exception to the logger for a clean logging.</p><p>This is a &quot;driver-only&quot; logger and is not intended at logging concurrent
-actions from executors.</p><p>Produced reports can easily be inserted in a notification email whenerver
+actions from executors.</p><p>Produced reports can easily be inserted in a notification email whenever
 the job fails, which saves a lot of time to maintainers operating on heavy
-production environements.</p><p>The produced persisted report is also a way for downstream jobs to know the
+production environments.</p><p>The produced persisted report is also a way for downstream jobs to know the
 status of their input data.</p><p>Let's go through a simple Spark job example monitored with this Monitor
 facility:</p><pre>Monitor.setTitle(<span class="lit">"My job title"</span>)
 Monitor.addDescription(
@@ -204,7 +222,7 @@ <h4 class="signature">
       Test(<span class="lit">"Nbr of output records"</span>, processedData.count(), SUPERIOR_THAN, <span class="num">10</span>e6d, NBR),
       Test(<span class="lit">"Some pct of invalid output"</span>, your_complex_kpi, INFERIOR_THAN, <span class="num">3</span>, PCT)
     ),
-    <span class="lit">"My pipeline descirption"</span>
+    <span class="lit">"My pipeline description"</span>
   )
 
   <span class="kw">if</span> (outputIsValid)
@@ -212,9 +230,9 @@ <h4 class="signature">
 
 } <span class="kw">catch</span> {
   <span class="kw">case</span> iie: InvalidInputException <span class="kw">=&gt;</span>
-    Monitor.error(iie, <span class="lit">"My pipeline descirption"</span>, diagnostic = <span class="lit">"No input data!"</span>)
+    Monitor.error(iie, <span class="lit">"My pipeline description"</span>, diagnostic = <span class="lit">"No input data!"</span>)
   <span class="kw">case</span> e: Throwable <span class="kw">=&gt;</span>
-    Monitor.error(e, <span class="lit">"My pipeline descirption"</span>) <span class="cmt">// whatever unexpected error</span>
+    Monitor.error(e, <span class="lit">"My pipeline description"</span>) <span class="cmt">// whatever unexpected error</span>
 }
 
 <span class="kw">if</span> (Monitor.isSuccess()) {
@@ -226,7 +244,7 @@ <h4 class="signature">
 <span class="cmt">// HDFS (this saves the logs in the folder set with Monitor.setLogFolder):</span>
 Monitor.store()
 
-<span class="cmt">// At the end of the job, if the job isn't successfull, you might want to</span>
+<span class="cmt">// At the end of the job, if the job isn't successful, you might want to</span>
 <span class="cmt">// crash it (for instance to get a notification from your scheduler):</span>
 <span class="kw">if</span> (!Monitor.isSuccess()) <span class="kw">throw</span> <span class="kw">new</span> Exception() <span class="cmt">// or send an email, or ...</span></pre><p>At any time during the job, logs can be accessed from file
 path/to/log/folder/current.ongoing</p><p>If we were to read the stored report after this simple pipeline, here are
@@ -234,8 +252,8 @@ <h4 class="signature">
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">23</span>] My pipeline descirption: failed
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">23</span>] My pipeline description: failed
   Diagnostic: No input data!
     org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: hdfs:<span class="cmt">//my/hdfs/input/path</span>
     at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:<span class="num">285</span>)
@@ -245,18 +263,18 @@ <h4 class="signature">
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">36</span>] My pipeline descirption: failed
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">36</span>] My pipeline description: failed
     java.lang.NumberFormatException: For input string: <span class="lit">"a"</span>
     java.lang.NumberFormatException.forInputString(NumberFormatException.java:<span class="num">65</span>)
     java.lang.Integer.parseInt(Integer.java:<span class="num">492</span>)
     ...
-[<span class="num">10</span>:<span class="num">36</span>] Duration: <span class="num">00</span>:<span class="num">13</span>:<span class="num">47</span></pre><p>Another scenario, successfull spark pipeline and KPIs are valid; all good!:</p><pre>          My job title
+[<span class="num">10</span>:<span class="num">36</span>] Duration: <span class="num">00</span>:<span class="num">13</span>:<span class="num">47</span></pre><p>Another scenario, successful spark pipeline and KPIs are valid; all good!:</p><pre>          My job title
 
 My job description (whatever you want); <span class="kw">for</span> instance:
 Documentation: https:<span class="cmt">//github.com/xavierguihot/spark_helper</span>
-[<span class="num">10</span>:<span class="num">23</span>] Begining
-[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">41</span>] My pipeline descirption: success
+[<span class="num">10</span>:<span class="num">23</span>] Beginning
+[<span class="num">10</span>:<span class="num">23</span>-<span class="num">10</span>:<span class="num">41</span>] My pipeline description: success
   KPI: Nbr of output records
     Value: <span class="num">14669071.0</span>
     Must be superior than <span class="num">10000000.0</span>
@@ -268,7 +286,7 @@ <h4 class="signature">
 [<span class="num">10</span>:<span class="num">41</span>-<span class="num">10</span>:<span class="num">42</span>] My second pipeline description: success
 [<span class="num">10</span>:<span class="num">42</span>] Duration: <span class="num">00</span>:<span class="num">19</span>:<span class="num">23</span></pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/monitoring/Monitor.scala">Monitor</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl></div>
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>would a State monad be appropriate?</p></span></dd></dl></div>
     </li><li name="com.spark_helper.SparkHelper" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
       <a id="SparkHelper"></a>
       <a id="SparkHelper:SparkHelper"></a>
@@ -287,19 +305,39 @@ <h4 class="signature">
     </span>
       <p class="shortcomment cmt">A facility to deal with RDD/file manipulations based on the Spark API.</p><div class="fullcomment"><div class="comment cmt"><p>A facility to deal with RDD/file manipulations based on the Spark API.</p><p>The goal is to remove the maximum of highly used low-level code from your
 spark job and replace it with methods fully tested whose name is
-self-explanatory/readable.</p><p>A few exemples:</p><pre><span class="cmt">// Same as SparkContext.saveAsTextFile, but the result is a single file:</span>
-SparkHelper.saveAsSingleTextFile(myOutputRDD, <span class="lit">"/my/output/file/path.txt"</span>)
-<span class="cmt">// Same as SparkContext.textFile, but instead of reading one record per</span>
-<span class="cmt">// line, it reads records spread over several lines.</span>
-<span class="cmt">// This way, xml, json, yml or any multi-line record file format can be used</span>
-<span class="cmt">// with Spark:</span>
-SparkHelper.textFileWithDelimiter(<span class="lit">"/my/input/folder/path"</span>, sparkContext, <span class="lit">"---\n"</span>)
-<span class="cmt">// Same as SparkContext.textFile, but instead of returning an RDD of</span>
-<span class="cmt">// records, it returns an RDD of tuples containing both the record and the</span>
-<span class="cmt">// path of the file it comes from:</span>
-SparkHelper.textFileWithFileName(<span class="lit">"folder"</span>, sparkContext)</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
+self-explanatory/readable.</p><p>A few examples:</p><pre><span class="kw">import</span> com.spark_helper.SparkHelper._
+
+<span class="cmt">// Same as rdd.saveAsTextFile("path"), but the result is a single file (while</span>
+<span class="cmt">// keeping the processing distributed):</span>
+rdd.saveAsSingleTextFile(<span class="lit">"/my/output/file/path.txt"</span>)
+rdd.saveAsSingleTextFile(<span class="lit">"/my/output/file/path.txt"</span>, classOf[BZip2Codec])
+
+<span class="cmt">// Same as sc.textFile("path"), but instead of reading one record per line (by</span>
+<span class="cmt">// splitting the input with \n), it splits the file in records based on a custom</span>
+<span class="cmt">// delimiter. This way, xml, json, yml or any multi-line record file format can</span>
+<span class="cmt">// be used with Spark:</span>
+sc.textFile(<span class="lit">"/my/input/folder/path"</span>, <span class="lit">"---\n"</span>) <span class="cmt">// for a yml file for instance</span>
+
+<span class="cmt">// Equivalent to rdd.flatMap(identity) for RDDs of Seqs or Options:</span>
+rdd.flatten
+
+<span class="cmt">// Equivalent to sc.textFile(), but for each line is tupled with its file path:</span>
+sc.textFileWithFileName(<span class="lit">"/my/input/folder/path"</span>)
+<span class="cmt">// which produces:</span>
+<span class="cmt">// RDD(("folder/file_1.txt", "record1fromfile1"), ("folder/file_1.txt", "record2fromfile1"),</span>
+<span class="cmt">//    ("folder/file_2.txt", "record1fromfile2"), ...)</span>
+
+<span class="cmt">// In the given folder, this generates one file per key in the given key/value</span>
+<span class="cmt">// RDD. Within each file (named from the key) are all values for this key:</span>
+rdd.saveAsTextFileByKey(<span class="lit">"/my/output/folder/path"</span>)
+
+<span class="cmt">// Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):</span>
+rdd.partialMap { <span class="kw">case</span> a <span class="kw">if</span> a % <span class="num">2</span> == <span class="num">0</span> <span class="kw">=&gt;</span> <span class="num">2</span> * a }
+
+<span class="cmt">// For when input files contain commas and textFile can't handle it:</span>
+sc.textFile(<span class="std">Seq</span>(<span class="lit">"path/hello,world.txt"</span>, <span class="lit">"path/hello_world.txt"</span>))</pre><p>Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
 /main/scala/com/spark_helper/SparkHelper.scala">SparkHelper</a>
-</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd></dl></div>
+</p></div><dl class="attributes block"> <dt>Since</dt><dd><p>2017-02</p></dd><dt>To do</dt><dd><span class="cmt"><p>sc.parallelize[T](elmts: T*) instead of sc.parallelize[T](elmts: Array[T])</p></span></dd></dl></div>
     </li><li name="com.spark_helper.monitoring" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
       <a id="monitoring"></a>
       <a id="monitoring:monitoring"></a>
diff --git a/docs/index.html b/docs/index.html
index eb6e8e8..4ec3b1d 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -25,7 +25,7 @@
       <div class="ui-west-center">
       <div id="filter">
           <div id="textfilter"></div>
-          <div id="letters"><span>#</span><a target="template" href="index/index-a.html">A</a><span>B</span><a target="template" href="index/index-c.html">C</a><a target="template" href="index/index-d.html">D</a><a target="template" href="index/index-e.html">E</a><a target="template" href="index/index-f.html">F</a><span>G</span><a target="template" href="index/index-h.html">H</a><a target="template" href="index/index-i.html">I</a><a target="template" href="index/index-j.html">J</a><a target="template" href="index/index-k.html">K</a><a target="template" href="index/index-l.html">L</a><a target="template" href="index/index-m.html">M</a><a target="template" href="index/index-n.html">N</a><span>O</span><a target="template" href="index/index-p.html">P</a><span>Q</span><a target="template" href="index/index-r.html">R</a><a target="template" href="index/index-s.html">S</a><a target="template" href="index/index-t.html">T</a><span>U</span><a target="template" href="index/index-v.html">V</a><a target="template" href="index/index-w.html">W</a><span>X</span><a target="template" href="index/index-y.html">Y</a><span>Z</span> – <span>deprecated</span></div>
+          <div id="letters"><span>#</span><a target="template" href="index/index-a.html">A</a><span>B</span><a target="template" href="index/index-c.html">C</a><a target="template" href="index/index-d.html">D</a><a target="template" href="index/index-e.html">E</a><a target="template" href="index/index-f.html">F</a><span>G</span><a target="template" href="index/index-h.html">H</a><a target="template" href="index/index-i.html">I</a><a target="template" href="index/index-j.html">J</a><a target="template" href="index/index-k.html">K</a><a target="template" href="index/index-l.html">L</a><a target="template" href="index/index-m.html">M</a><a target="template" href="index/index-n.html">N</a><a target="template" href="index/index-o.html">O</a><a target="template" href="index/index-p.html">P</a><span>Q</span><a target="template" href="index/index-r.html">R</a><a target="template" href="index/index-s.html">S</a><a target="template" href="index/index-t.html">T</a><span>U</span><a target="template" href="index/index-v.html">V</a><a target="template" href="index/index-w.html">W</a><span>X</span><a target="template" href="index/index-y.html">Y</a><span>Z</span> – <span>deprecated</span></div>
       </div>
       <div class="pack" id="tpl">
             
@@ -42,6 +42,18 @@
             <ol class="packages"> </ol>
           </li></ol>
           </li></ol>
+          </li><li class="pack" title="org">
+            <a class="tplshow" href="org/package.html" target="template">org</a>
+            <ol class="templates"></ol>
+            <ol class="packages"> <li class="pack" title="org.apache">
+            <a class="tplshow" href="org/apache/package.html" target="template">org.apache</a>
+            <ol class="templates"></ol>
+            <ol class="packages"> <li class="pack" title="org.apache.spark">
+            <a class="tplshow" href="org/apache/spark/package.html" target="template">org.apache.spark</a>
+            <ol class="templates"><li title="org.apache.spark.TextFileOverwrite"><a class="tplshow" href="org/apache/spark/TextFileOverwrite$.html" target="template"><span class="object">(object)</span><div class="placeholder"></div><span class="tplLink">TextFileOverwrite</span></a></li></ol>
+            <ol class="packages"> </ol>
+          </li></ol>
+          </li></ol>
           </li></ol>
           </div></div><script src="index.js"></script>
     </div>
diff --git a/docs/index.js b/docs/index.js
index 8f335a2..6668d87 100644
--- a/docs/index.js
+++ b/docs/index.js
@@ -1 +1 @@
-Index.PACKAGES = {"com" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/Monitor$.html", "name" : "com.spark_helper.Monitor"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"case class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}]};
\ No newline at end of file
+Index.PACKAGES = {"com.spark_helper.monitoring" : [{"object" : "com\/spark_helper\/monitoring\/EQUAL_TO$.html", "name" : "com.spark_helper.monitoring.EQUAL_TO"}, {"object" : "com\/spark_helper\/monitoring\/INFERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.INFERIOR_THAN"}, {"trait" : "com\/spark_helper\/monitoring\/KpiType.html", "name" : "com.spark_helper.monitoring.KpiType"}, {"object" : "com\/spark_helper\/monitoring\/NBR$.html", "name" : "com.spark_helper.monitoring.NBR"}, {"object" : "com\/spark_helper\/monitoring\/PCT$.html", "name" : "com.spark_helper.monitoring.PCT"}, {"object" : "com\/spark_helper\/monitoring\/SUPERIOR_THAN$.html", "name" : "com.spark_helper.monitoring.SUPERIOR_THAN"}, {"case class" : "com\/spark_helper\/monitoring\/Test.html", "name" : "com.spark_helper.monitoring.Test"}, {"trait" : "com\/spark_helper\/monitoring\/ThresholdType.html", "name" : "com.spark_helper.monitoring.ThresholdType"}], "org.apache.spark" : [{"object" : "org\/apache\/spark\/TextFileOverwrite$.html", "name" : "org.apache.spark.TextFileOverwrite"}], "org.apache" : [], "com.spark_helper" : [{"object" : "com\/spark_helper\/DateHelper$.html", "name" : "com.spark_helper.DateHelper"}, {"object" : "com\/spark_helper\/HdfsHelper$.html", "name" : "com.spark_helper.HdfsHelper"}, {"object" : "com\/spark_helper\/Monitor$.html", "name" : "com.spark_helper.Monitor"}, {"object" : "com\/spark_helper\/SparkHelper$.html", "name" : "com.spark_helper.SparkHelper"}], "org" : [], "com" : []};
\ No newline at end of file
diff --git a/docs/index/index-a.html b/docs/index/index-a.html
index 557f98b..5d8c458 100644
--- a/docs/index/index-a.html
+++ b/docs/index/index-a.html
@@ -16,6 +16,9 @@
     </div><div class="entry">
       <div class="name">addDescription</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
+    </div><div class="entry">
+      <div class="name">apache</div>
+      <div class="occurrences"><a href="../org/package.html" class="extype" name="org">org</a> </div>
     </div><div class="entry">
       <div class="name">appendFooter</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
diff --git a/docs/index/index-d.html b/docs/index/index-d.html
index 860d355..a6dd1d1 100644
--- a/docs/index/index-d.html
+++ b/docs/index/index-d.html
@@ -19,12 +19,21 @@
     </div><div class="entry">
       <div class="name">dayOfWeek</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">daysAfter</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$IntExtensions.html" class="extype" name="com.spark_helper.DateHelper.IntExtensions">IntExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">daysAgo</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$IntExtensions.html" class="extype" name="com.spark_helper.DateHelper.IntExtensions">IntExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">daysBefore</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$IntExtensions.html" class="extype" name="com.spark_helper.DateHelper.IntExtensions">IntExtensions</a> </div>
     </div><div class="entry">
       <div class="name">daysBetween</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
     </div><div class="entry">
       <div class="name">decreaseCoalescence</div>
-      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$SparkContextExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SparkContextExtensions">SparkContextExtensions</a> </div>
     </div><div class="entry">
       <div class="name">deleteFile</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
diff --git a/docs/index/index-f.html b/docs/index/index-f.html
index e17cd19..3e21430 100644
--- a/docs/index/index-f.html
+++ b/docs/index/index-f.html
@@ -19,6 +19,9 @@
     </div><div class="entry">
       <div class="name">fileModificationDateTime</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">flatten</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$OptionRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions">OptionRDDExtensions</a> <a href="../com/spark_helper/SparkHelper$$SeqRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions">SeqRDDExtensions</a> </div>
     </div><div class="entry">
       <div class="name">folderExists</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
diff --git a/docs/index/index-i.html b/docs/index/index-i.html
index 07f867e..d1f7b53 100644
--- a/docs/index/index-i.html
+++ b/docs/index/index-i.html
@@ -13,6 +13,15 @@
         <body><div class="entry">
       <div class="name">INFERIOR_THAN</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
+    </div><div class="entry">
+      <div class="name">IntExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">int</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$IntExtensions.html" class="extype" name="com.spark_helper.DateHelper.IntExtensions">IntExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">isCompliantWith</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">isDateCompliantWithFormat</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
diff --git a/docs/index/index-n.html b/docs/index/index-n.html
index 914674a..128fca2 100644
--- a/docs/index/index-n.html
+++ b/docs/index/index-n.html
@@ -13,6 +13,9 @@
         <body><div class="entry">
       <div class="name">NBR</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
+    </div><div class="entry">
+      <div class="name">nDaysAfter</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
     </div><div class="entry">
       <div class="name">nDaysAfterDate</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
@@ -36,7 +39,7 @@
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
     </div><div class="entry">
       <div class="name">nextDay</div>
-      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> <a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">now</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
diff --git a/docs/index/index-o.html b/docs/index/index-o.html
new file mode 100644
index 0000000..4a559d5
--- /dev/null
+++ b/docs/index/index-o.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title></title>
+          <meta name="description" content="" />
+          <meta name="keywords" content="" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../lib/ref-index.css" media="screen" type="text/css" rel="stylesheet" />
+      <script type="text/javascript" src="../lib/jquery.js"></script>
+    
+        </head>
+        <body><div class="entry">
+      <div class="name">OptionRDDExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">org</div>
+      <div class="occurrences"><a href="../package.html" class="extype" name="_root_">root</a> </div>
+    </div></body>
+      </html>
diff --git a/docs/index/index-p.html b/docs/index/index-p.html
index c004302..bdee856 100644
--- a/docs/index/index-p.html
+++ b/docs/index/index-p.html
@@ -13,9 +13,15 @@
         <body><div class="entry">
       <div class="name">PCT</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
+    </div><div class="entry">
+      <div class="name">PairRDDExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">partialMap</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$RDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.RDDExtensions">RDDExtensions</a> </div>
     </div><div class="entry">
       <div class="name">previousDay</div>
-      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> <a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">purgeFolder</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
diff --git a/docs/index/index-r.html b/docs/index/index-r.html
index 8ef3476..317ec21 100644
--- a/docs/index/index-r.html
+++ b/docs/index/index-r.html
@@ -11,6 +11,12 @@
     
         </head>
         <body><div class="entry">
+      <div class="name">RDDExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">rdd</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$OptionRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.OptionRDDExtensions">OptionRDDExtensions</a> <a href="../com/spark_helper/SparkHelper$$PairRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.PairRDDExtensions">PairRDDExtensions</a> <a href="../com/spark_helper/SparkHelper$$RDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.RDDExtensions">RDDExtensions</a> <a href="../com/spark_helper/SparkHelper$$SeqRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SeqRDDExtensions">SeqRDDExtensions</a> <a href="../com/spark_helper/SparkHelper$$StringRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.StringRDDExtensions">StringRDDExtensions</a> </div>
+    </div><div class="entry">
       <div class="name">reformatDate</div>
       <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
     </div></body>
diff --git a/docs/index/index-s.html b/docs/index/index-s.html
index f70ca30..c0e0953 100644
--- a/docs/index/index-s.html
+++ b/docs/index/index-s.html
@@ -11,32 +11,86 @@
     
         </head>
         <body><div class="entry">
+      <div class="name">SPARK_BRANCH</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
+      <div class="name">SPARK_BUILD_DATE</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
+      <div class="name">SPARK_BUILD_USER</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
+      <div class="name">SPARK_REPO_URL</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
+      <div class="name">SPARK_REVISION</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
+      <div class="name">SPARK_VERSION</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
+    </div><div class="entry">
       <div class="name">SUPERIOR_THAN</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
+    </div><div class="entry">
+      <div class="name">SeqExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">SeqRDDExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">SparkContextExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
     </div><div class="entry">
       <div class="name">SparkHelper</div>
       <div class="occurrences"><a href="../com/spark_helper/package.html" class="extype" name="com.spark_helper">spark_helper</a> </div>
     </div><div class="entry">
-      <div class="name">saveAsSingleTextFile</div>
+      <div class="name">StringExtensions</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> <a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">StringRDDExtensions</div>
       <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">saveAsSingleTextFile</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$StringRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.StringRDDExtensions">StringRDDExtensions</a> </div>
     </div><div class="entry">
       <div class="name">saveAsTextFileAndCoalesce</div>
-      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$StringRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.StringRDDExtensions">StringRDDExtensions</a> </div>
     </div><div class="entry">
       <div class="name">saveAsTextFileByKey</div>
-      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$PairRDDExtensions.html" class="extype" name="com.spark_helper.SparkHelper.PairRDDExtensions">PairRDDExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">sc</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$SparkContextExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SparkContextExtensions">SparkContextExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">seq</div>
+      <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$$SeqExtensions.html" class="extype" name="com.spark_helper.HdfsHelper.SeqExtensions">SeqExtensions</a> </div>
+    </div><div class="entry">
+      <div class="name">setConf</div>
+      <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">setFileSystem</div>
+      <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
+    </div><div class="entry">
+      <div class="name">setFormat</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$.html" class="extype" name="com.spark_helper.DateHelper">DateHelper</a> </div>
     </div><div class="entry">
       <div class="name">setLogFolder</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
     </div><div class="entry">
       <div class="name">setTitle</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
+    </div><div class="entry">
+      <div class="name">spark</div>
+      <div class="occurrences"><a href="../org/apache/package.html" class="extype" name="org.apache">apache</a> </div>
     </div><div class="entry">
       <div class="name">spark_helper</div>
       <div class="occurrences"><a href="../com/package.html" class="extype" name="com">com</a> </div>
     </div><div class="entry">
       <div class="name">store</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
+    </div><div class="entry">
+      <div class="name">string</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> <a href="../com/spark_helper/HdfsHelper$$StringExtensions.html" class="extype" name="com.spark_helper.HdfsHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">success</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
diff --git a/docs/index/index-t.html b/docs/index/index-t.html
index bb8d1a4..c58eef1 100644
--- a/docs/index/index-t.html
+++ b/docs/index/index-t.html
@@ -13,18 +13,24 @@
         <body><div class="entry">
       <div class="name">Test</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
+    </div><div class="entry">
+      <div class="name">TextFileOverwrite</div>
+      <div class="occurrences"><a href="../org/apache/spark/package.html" class="extype" name="org.apache.spark">spark</a> </div>
     </div><div class="entry">
       <div class="name">ThresholdType</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/package.html" class="extype" name="com.spark_helper.monitoring">monitoring</a> </div>
     </div><div class="entry">
-      <div class="name">textFileWithDelimiter</div>
-      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+      <div class="name">textFile</div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$SparkContextExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SparkContextExtensions">SparkContextExtensions</a> <a href="../org/apache/spark/TextFileOverwrite$.html" class="extype" name="org.apache.spark.TextFileOverwrite">TextFileOverwrite</a> </div>
     </div><div class="entry">
       <div class="name">textFileWithFileName</div>
-      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$.html" class="extype" name="com.spark_helper.SparkHelper">SparkHelper</a> </div>
+      <div class="occurrences"><a href="../com/spark_helper/SparkHelper$$SparkContextExtensions.html" class="extype" name="com.spark_helper.SparkHelper.SparkContextExtensions">SparkContextExtensions</a> </div>
     </div><div class="entry">
       <div class="name">thresholdType</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/Test.html" class="extype" name="com.spark_helper.monitoring.Test">Test</a> </div>
+    </div><div class="entry">
+      <div class="name">to</div>
+      <div class="occurrences"><a href="../com/spark_helper/DateHelper$$StringExtensions.html" class="extype" name="com.spark_helper.DateHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">toString</div>
       <div class="occurrences"><a href="../com/spark_helper/monitoring/Test.html" class="extype" name="com.spark_helper.monitoring.Test">Test</a> </div>
diff --git a/docs/index/index-w.html b/docs/index/index-w.html
index 231f199..f7f4a02 100644
--- a/docs/index/index-w.html
+++ b/docs/index/index-w.html
@@ -13,6 +13,9 @@
         <body><div class="entry">
       <div class="name">withPurge</div>
       <div class="occurrences"><a href="../com/spark_helper/Monitor$.html" class="extype" name="com.spark_helper.Monitor">Monitor</a> </div>
+    </div><div class="entry">
+      <div class="name">writeToHdfs</div>
+      <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$$SeqExtensions.html" class="extype" name="com.spark_helper.HdfsHelper.SeqExtensions">SeqExtensions</a> <a href="../com/spark_helper/HdfsHelper$$StringExtensions.html" class="extype" name="com.spark_helper.HdfsHelper.StringExtensions">StringExtensions</a> </div>
     </div><div class="entry">
       <div class="name">writeToHdfsFile</div>
       <div class="occurrences"><a href="../com/spark_helper/HdfsHelper$.html" class="extype" name="com.spark_helper.HdfsHelper">HdfsHelper</a> </div>
diff --git a/docs/org/apache/package.html b/docs/org/apache/package.html
new file mode 100644
index 0000000..724d793
--- /dev/null
+++ b/docs/org/apache/package.html
@@ -0,0 +1,118 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>apache - org.apache</title>
+          <meta name="description" content="apache - org.apache" />
+          <meta name="keywords" content="apache org.apache" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../lib/template.js"></script>
+      <script type="text/javascript" src="../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../index.html';
+            var hash = 'org.apache.package';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="value">
+      <div id="definition">
+        <img alt="Package" src="../../lib/package_big.png" />
+        <p id="owner"><a href="../package.html" class="extype" name="org">org</a></p>
+        <h1>apache</h1><span class="permalink">
+      <a href="../../index.html#org.apache.package" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <span class="name">apache</span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        
+        
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="org.apache.spark" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="spark"></a>
+      <a id="spark:spark"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <a href="spark/package.html"><span class="name">spark</span></a>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../index.html#org.apache.package@spark" title="Permalink" target="_top">
+        <img src="../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/org/apache/spark/TextFileOverwrite$.html b/docs/org/apache/spark/TextFileOverwrite$.html
new file mode 100644
index 0000000..908c792
--- /dev/null
+++ b/docs/org/apache/spark/TextFileOverwrite$.html
@@ -0,0 +1,499 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>TextFileOverwrite - org.apache.spark.TextFileOverwrite</title>
+          <meta name="description" content="TextFileOverwrite - org.apache.spark.TextFileOverwrite" />
+          <meta name="keywords" content="TextFileOverwrite org.apache.spark.TextFileOverwrite" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../../lib/template.js"></script>
+      <script type="text/javascript" src="../../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../../index.html';
+            var hash = 'org.apache.spark.TextFileOverwrite$';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="value">
+      <div id="definition">
+        <img alt="Object" src="../../../lib/object_big.png" />
+        <p id="owner"><a href="../../package.html" class="extype" name="org">org</a>.<a href="../package.html" class="extype" name="org.apache">apache</a>.<a href="package.html" class="extype" name="org.apache.spark">spark</a></p>
+        <h1>TextFileOverwrite</h1><h3><span class="morelinks"><div>Related Doc:
+            <a href="package.html" class="extype" name="org.apache.spark">package spark</a>
+          </div></span></h3><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">object</span>
+      </span>
+      <span class="symbol">
+        <span class="name">TextFileOverwrite</span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="org.apache.spark.TextFileOverwrite"><span>TextFileOverwrite</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="scala.AnyRef#!=" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="!=(x$1:Any):Boolean"></a>
+      <a id="!=(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $bang$eq" class="name">!=</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@!=(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef###" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="##():Int"></a>
+      <a id="##():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $hash$hash" class="name">##</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@##():Int" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#==" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="==(x$1:Any):Boolean"></a>
+      <a id="==(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span title="gt4s: $eq$eq" class="name">==</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@==(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#asInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="asInstanceOf[T0]:T0"></a>
+      <a id="asInstanceOf[T0]:T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">asInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Any.asInstanceOf.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@asInstanceOf[T0]:T0" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#clone" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="clone():Object"></a>
+      <a id="clone():AnyRef"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">clone</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.AnyRef">AnyRef</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@clone():Object" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.CloneNotSupportedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#eq" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="eq(x$1:AnyRef):Boolean"></a>
+      <a id="eq(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">eq</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@eq(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#equals" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="equals(x$1:Any):Boolean"></a>
+      <a id="equals(Any):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">equals</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Any">Any</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@equals(x$1:Any):Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#finalize" visbl="prt" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="finalize():Unit"></a>
+      <a id="finalize():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">finalize</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@finalize():Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Attributes</dt><dd>protected[<a href="../../../java$lang.html" class="extype" name="java.lang">java.lang</a>] </dd><dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="symbol">classOf[java.lang.Throwable]</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#getClass" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="getClass():Class[_]"></a>
+      <a id="getClass():Class[_]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">getClass</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.Class">Class</span>[_]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@getClass():Class[_]" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#hashCode" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="hashCode():Int"></a>
+      <a id="hashCode():Int"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">hashCode</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Int">Int</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@hashCode():Int" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.Any#isInstanceOf" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <a id="isInstanceOf[T0]:Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">isInstanceOf</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@isInstanceOf[T0]:Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>Any</dd></dl></div>
+    </li><li name="scala.AnyRef#ne" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="ne(x$1:AnyRef):Boolean"></a>
+      <a id="ne(AnyRef):Boolean"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">ne</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.AnyRef">AnyRef</span></span>)</span><span class="result">: <span class="extype" name="scala.Boolean">Boolean</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@ne(x$1:AnyRef):Boolean" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notify" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notify():Unit"></a>
+      <a id="notify():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notify</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@notify():Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#notifyAll" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="notifyAll():Unit"></a>
+      <a id="notifyAll():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">notifyAll</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@notifyAll():Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="scala.AnyRef#synchronized" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="synchronized[T0](x$1:=&gt;T0):T0"></a>
+      <a id="synchronized[T0](⇒T0):T0"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">synchronized</span><span class="tparams">[<span name="T0">T0</span>]</span><span class="params">(<span name="arg0">arg0: ⇒ <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>)</span><span class="result">: <span class="extype" name="java.lang.AnyRef.synchronized.T0">T0</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@synchronized[T0](x$1:=&gt;T0):T0" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd></dl></div>
+    </li><li name="org.apache.spark.TextFileOverwrite#textFile" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="textFile(paths:Seq[String],minPartitions:Int,sc:org.apache.spark.SparkContext):org.apache.spark.rdd.RDD[String]"></a>
+      <a id="textFile(Seq[String],Int,SparkContext):RDD[String]"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">textFile</span><span class="params">(<span name="paths">paths: <span class="extype" name="scala.Seq">Seq</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>, <span name="minPartitions">minPartitions: <span class="extype" name="scala.Int">Int</span></span>, <span name="sc">sc: <span class="extype" name="org.apache.spark.SparkContext">SparkContext</span></span>)</span><span class="result">: <span class="extype" name="org.apache.spark.rdd.RDD">RDD</span>[<span class="extype" name="scala.Predef.String">String</span>]</span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@textFile(paths:Seq[String],minPartitions:Int,sc:org.apache.spark.SparkContext):org.apache.spark.rdd.RDD[String]" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="scala.AnyRef#toString" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="toString():String"></a>
+      <a id="toString():String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">toString</span><span class="params">()</span><span class="result">: <span class="extype" name="java.lang.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@toString():String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef → Any</dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait():Unit"></a>
+      <a id="wait():Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">()</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@wait():Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long,x$2:Int):Unit"></a>
+      <a id="wait(Long,Int):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>, <span name="arg1">arg1: <span class="extype" name="scala.Int">Int</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@wait(x$1:Long,x$2:Int):Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li><li name="scala.AnyRef#wait" visbl="pub" data-isabs="false" fullComment="yes" group="Ungrouped">
+      <a id="wait(x$1:Long):Unit"></a>
+      <a id="wait(Long):Unit"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier">final </span>
+        <span class="kind">def</span>
+      </span>
+      <span class="symbol">
+        <span class="name">wait</span><span class="params">(<span name="arg0">arg0: <span class="extype" name="scala.Long">Long</span></span>)</span><span class="result">: <span class="extype" name="scala.Unit">Unit</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.TextFileOverwrite$@wait(x$1:Long):Unit" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      <div class="fullcomment"><dl class="attributes block"> <dt>Definition Classes</dt><dd>AnyRef</dd><dt>Annotations</dt><dd>
+                <span class="name">@throws</span><span class="args">(<span>
+      
+      <span class="defval" name="classOf[java.lang.InterruptedException]">...</span>
+    </span>)</span>
+              
+        </dd></dl></div>
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/org/apache/spark/package.html b/docs/org/apache/spark/package.html
new file mode 100644
index 0000000..9ae6205
--- /dev/null
+++ b/docs/org/apache/spark/package.html
@@ -0,0 +1,246 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>spark - org.apache.spark</title>
+          <meta name="description" content="spark - org.apache.spark" />
+          <meta name="keywords" content="spark org.apache.spark" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../../../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../../../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../../../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../../../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../../../lib/template.js"></script>
+      <script type="text/javascript" src="../../../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../../../index.html';
+            var hash = 'org.apache.spark.package';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="value">
+      <div id="definition">
+        <img alt="Package" src="../../../lib/package_big.png" />
+        <p id="owner"><a href="../../package.html" class="extype" name="org">org</a>.<a href="../package.html" class="extype" name="org.apache">apache</a></p>
+        <h1>spark</h1><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <span class="name">spark</span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"><div class="toggleContainer block">
+          <span class="toggle">Linear Supertypes</span>
+          <div class="superTypes hiddenContent"><span class="extype" name="scala.AnyRef">AnyRef</span>, <span class="extype" name="scala.Any">Any</span></div>
+        </div></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        <div id="order">
+              <span class="filtertype">Ordering</span>
+              <ol>
+                
+                <li class="alpha in"><span>Alphabetic</span></li>
+                <li class="inherit out"><span>By Inheritance</span></li>
+              </ol>
+            </div>
+        <div id="ancestors">
+                <span class="filtertype">Inherited<br />
+                </span>
+                <ol id="linearization">
+                  <li class="in" name="org.apache.spark"><span>spark</span></li><li class="in" name="scala.AnyRef"><span>AnyRef</span></li><li class="in" name="scala.Any"><span>Any</span></li>
+                </ol>
+              </div><div id="ancestors">
+            <span class="filtertype"></span>
+            <ol>
+              <li class="hideall out"><span>Hide All</span></li>
+              <li class="showall in"><span>Show All</span></li>
+            </ol>
+          </div>
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="org.apache.spark#SPARK_BRANCH" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_BRANCH:String"></a>
+      <a id="SPARK_BRANCH:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_BRANCH</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_BRANCH:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark#SPARK_BUILD_DATE" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_BUILD_DATE:String"></a>
+      <a id="SPARK_BUILD_DATE:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_BUILD_DATE</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_BUILD_DATE:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark#SPARK_BUILD_USER" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_BUILD_USER:String"></a>
+      <a id="SPARK_BUILD_USER:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_BUILD_USER</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_BUILD_USER:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark#SPARK_REPO_URL" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_REPO_URL:String"></a>
+      <a id="SPARK_REPO_URL:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_REPO_URL</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_REPO_URL:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark#SPARK_REVISION" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_REVISION:String"></a>
+      <a id="SPARK_REVISION:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_REVISION</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_REVISION:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark#SPARK_VERSION" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="SPARK_VERSION:String"></a>
+      <a id="SPARK_VERSION:String"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">val</span>
+      </span>
+      <span class="symbol">
+        <span class="name">SPARK_VERSION</span><span class="result">: <span class="extype" name="scala.Predef.String">String</span></span>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@SPARK_VERSION:String" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li><li name="org.apache.spark.TextFileOverwrite" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="TextFileOverwrite"></a>
+      <a id="TextFileOverwrite:TextFileOverwrite"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">object</span>
+      </span>
+      <span class="symbol">
+        <a href="TextFileOverwrite$.html"><span class="name">TextFileOverwrite</span></a>
+      </span>
+      </h4><span class="permalink">
+      <a href="../../../index.html#org.apache.spark.package@TextFileOverwrite" title="Permalink" target="_top">
+        <img src="../../../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        <div class="parent" name="scala.AnyRef">
+              <h3>Inherited from <span class="extype" name="scala.AnyRef">AnyRef</span></h3>
+            </div><div class="parent" name="scala.Any">
+              <h3>Inherited from <span class="extype" name="scala.Any">Any</span></h3>
+            </div>
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/org/package.html b/docs/org/package.html
new file mode 100644
index 0000000..329e4fd
--- /dev/null
+++ b/docs/org/package.html
@@ -0,0 +1,118 @@
+<!DOCTYPE html >
+<html>
+        <head>
+          <title>org - org</title>
+          <meta name="description" content="org - org" />
+          <meta name="keywords" content="org org" />
+          <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
+          
+      <link href="../lib/template.css" media="screen" type="text/css" rel="stylesheet" />
+      <link href="../lib/diagrams.css" media="screen" type="text/css" rel="stylesheet" id="diagrams-css" />
+      <script type="text/javascript" src="../lib/jquery.js" id="jquery-js"></script>
+      <script type="text/javascript" src="../lib/jquery-ui.js"></script>
+      <script type="text/javascript" src="../lib/template.js"></script>
+      <script type="text/javascript" src="../lib/tools.tooltip.js"></script>
+      
+      <script type="text/javascript">
+         if(top === self) {
+            var url = '../index.html';
+            var hash = 'org.package';
+            var anchor = window.location.hash;
+            var anchor_opt = '';
+            if (anchor.length >= 1)
+              anchor_opt = '@' + anchor.substring(1);
+            window.location.href = url + '#' + hash + anchor_opt;
+         }
+   	  </script>
+    
+        </head>
+        <body class="value">
+      <div id="definition">
+        <img alt="Package" src="../lib/package_big.png" />
+        
+        <h1>org</h1><span class="permalink">
+      <a href="../index.html#org.package" title="Permalink" target="_top">
+        <img src="../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      </div>
+
+      <h4 id="signature" class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <span class="name">org</span>
+      </span>
+      </h4>
+      
+          <div id="comment" class="fullcommenttop"></div>
+        
+
+      <div id="mbrsel">
+        <div id="textfilter"><span class="pre"></span><span class="input"><input id="mbrsel-input" type="text" accesskey="/" /></span><span class="post"></span></div>
+        
+        
+        <div id="visbl">
+            <span class="filtertype">Visibility</span>
+            <ol><li class="public in"><span>Public</span></li><li class="all out"><span>All</span></li></ol>
+          </div>
+      </div>
+
+      <div id="template">
+        <div id="allMembers">
+        
+
+        
+
+        
+
+        <div id="values" class="values members">
+              <h3>Value Members</h3>
+              <ol><li name="org.apache" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="apache"></a>
+      <a id="apache:apache"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <a href="apache/package.html"><span class="name">apache</span></a>
+      </span>
+      </h4><span class="permalink">
+      <a href="../index.html#org.package@apache" title="Permalink" target="_top">
+        <img src="../lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
+    </li></ol>
+            </div>
+
+        
+
+        
+        </div>
+
+        <div id="inheritedMembers">
+        
+        
+        </div>
+
+        <div id="groupedMembers">
+        <div class="group" name="Ungrouped">
+              <h3>Ungrouped</h3>
+              
+            </div>
+        </div>
+
+      </div>
+
+      <div id="tooltip"></div>
+
+      <div id="footer">  </div>
+
+
+    </body>
+      </html>
diff --git a/docs/package.html b/docs/package.html
index b941069..a6a120c 100644
--- a/docs/package.html
+++ b/docs/package.html
@@ -87,6 +87,23 @@ <h4 class="signature">
       </a>
     </span>
       
+    </li><li name="_root_.org" visbl="pub" data-isabs="false" fullComment="no" group="Ungrouped">
+      <a id="org"></a>
+      <a id="org:org"></a>
+      <h4 class="signature">
+      <span class="modifier_kind">
+        <span class="modifier"></span>
+        <span class="kind">package</span>
+      </span>
+      <span class="symbol">
+        <a href="org/package.html"><span class="name">org</span></a>
+      </span>
+      </h4><span class="permalink">
+      <a href="index.html#package@org" title="Permalink" target="_top">
+        <img src="lib/permalink.png" alt="Permalink" />
+      </a>
+    </span>
+      
     </li></ol>
             </div>
 
diff --git a/src/main/scala/com/spark_helper/DateHelper.scala b/src/main/scala/com/spark_helper/DateHelper.scala
index d393718..229a648 100644
--- a/src/main/scala/com/spark_helper/DateHelper.scala
+++ b/src/main/scala/com/spark_helper/DateHelper.scala
@@ -15,14 +15,31 @@ import scala.util.Try
   * A few examples:
   *
   * {{{
-  * assert(DateHelper.daysBetween("20161230", "20170101") == List("20161230", "20161231", "20170101"))
-  * assert(DateHelper.today() == "20170310") // If today's "20170310"
-  * assert(DateHelper.yesterday() == "20170309") // If today's "20170310"
-  * assert(DateHelper.reformatDate("20170327", "yyyyMMdd", "yyMMdd") == "170327")
-  * assert(DateHelper.now("HH:mm") == "10:24")
-  * assert(DateHelper.currentTimestamp() == "1493105229736")
-  * assert(DateHelper.nDaysBefore(3) == "20170307") // If today's "20170310"
-  * assert(DateHelper.nDaysAfterDate(3, "20170307") == "20170310")
+  * import com.spark_helper.DateHelper
+  *
+  * DateHelper.daysBetween("20161230", "20170101") // List("20161230", "20161231", "20170101")
+  * DateHelper.today // "20170310"
+  * DateHelper.yesterday // "20170309"
+  * DateHelper.reformatDate("20170327", "yyyyMMdd", "yyMMdd") // "170327"
+  * DateHelper.now("HH:mm") // "10:24"
+  * DateHelper.currentTimestamp // "1493105229736"
+  * DateHelper.nDaysBefore(3) // "20170307"
+  * DateHelper.nDaysAfterDate(3, "20170307") // "20170310"
+  * DateHelper.nextDay("20170310") // "20170311"
+  * DateHelper.nbrOfDaysSince("20170302") // 8
+  * DateHelper.nbrOfDaysBetween("20170327", "20170401") // 5
+  * DateHelper.dayOfWeek("20160614") // 2
+  *
+  * import com.spark_helper.DateHelper._
+  *
+  * 2.daysAgo // "20170308"
+  * "20161230" to "20170101" // List("20161230", "20161231", "20170101")
+  * 3.daysBefore("20170310") // "20170307"
+  * 5.daysAfter // "20170315"
+  * 4.daysAfter("20170310") // "20170314"
+  * "20170302".isCompliantWith("yyyyMMdd")
+  * "20170310".nextDay // "20170311"
+  * "20170310".previousDay // "20170309"
   * }}}
   *
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src
diff --git a/src/main/scala/com/spark_helper/HdfsHelper.scala b/src/main/scala/com/spark_helper/HdfsHelper.scala
index b4cd82a..f2a1364 100644
--- a/src/main/scala/com/spark_helper/HdfsHelper.scala
+++ b/src/main/scala/com/spark_helper/HdfsHelper.scala
@@ -42,32 +42,33 @@ import com.typesafe.config.{Config, ConfigFactory}
   * import com.spark_helper.HdfsHelper
   *
   * // A bunch of methods wrapping the FileSystem API, such as:
-  * HdfsHelper.fileExists("my/hdfs/file/path.txt")
-  * assert(HdfsHelper.listFileNamesInFolder("my/folder/path") == List("file_name_1.txt", "file_name_2.csv"))
-  * assert(HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") == "20170306")
-  * assert(HdfsHelper.nbrOfDaysSinceFileWasLastModified("my/hdfs/file/path.txt") == 3)
-  * HdfsHelper.deleteFile("my/hdfs/file/path.csv")
-  * HdfsHelper.moveFolder("my/hdfs/folder")
+  * HdfsHelper.fileExists("my/hdfs/file/path.txt") // HdfsHelper.folderExists("my/hdfs/folder")
+  * HdfsHelper.listFileNamesInFolder("my/folder/path") // List("file_name_1.txt", "file_name_2.csv")
+  * HdfsHelper.fileModificationDate("my/hdfs/file/path.txt") // "20170306"
+  * HdfsHelper.nbrOfDaysSinceFileWasLastModified("my/hdfs/file/path.txt") // 3
+  * HdfsHelper.deleteFile("my/hdfs/file/path.csv") // HdfsHelper.deleteFolder("my/hdfs/folder")
+  * HdfsHelper.moveFolder("old/path", "new/path") // HdfsHelper.moveFile("old/path.txt", "new/path.txt")
+  * HdfsHelper.createEmptyHdfsFile("/some/hdfs/file/path.token") // HdfsHelper.createFolder("my/hdfs/folder")
+  *
+  * // File content helpers:
   * HdfsHelper.compressFile("hdfs/path/to/uncompressed_file.txt", classOf[GzipCodec])
-  * HdfsHelper.appendHeader("my/hdfs/file/path.csv", "column0,column1")
+  * HdfsHelper.appendHeader("my/hdfs/file/path.csv", "colum0,column1")
   *
   * // Some Xml/Typesafe helpers for hadoop as well:
-  * HdfsHelper.isHdfsXmlCompliantWithXsd(
-  *   "my/hdfs/file/path.xml", getClass.getResource("/some_xml.xsd"))
+  * HdfsHelper.isHdfsXmlCompliantWithXsd("my/hdfs/file/path.xml", getClass.getResource("/some_xml.xsd"))
   * HdfsHelper.loadXmlFileFromHdfs("my/hdfs/file/path.xml")
   *
-  * // Very handy to load a config (typesafe format) stored on hdfs at the
-  * // beginning of a spark job:
+  * // Very handy to load a config (typesafe format) stored on hdfs at the beginning of a spark job:
   * HdfsHelper.loadTypesafeConfigFromHdfs("my/hdfs/file/path.conf"): Config
   *
-  * // In order to write small amount of data in a file on hdfs without the
-  * // whole spark stack:
-  * HdfsHelper.writeToHdfsFile(
-  *   Array("some", "relatively small", "text"),
-  *   "/some/hdfs/file/path.txt")
+  * // In order to write small amount of data in a file on hdfs without the whole spark stack:
+  * HdfsHelper.writeToHdfsFile(Array("some", "relatively small", "text"), "/some/hdfs/file/path.txt")
+  * // or:
+  * import com.spark_helper.HdfsHelper._
+  * Array("some", "relatively small", "text").writeToHdfs("/some/hdfs/file/path.txt")
+  * "hello world".writeToHdfs("/some/hdfs/file/path.txt")
   *
-  * // Deletes all files/folders in "hdfs/path/to/folder" for which the
-  * // timestamp is older than 10 days:
+  * // Deletes all files/folders in "hdfs/path/to/folder" for which the timestamp is older than 10 days:
   * HdfsHelper.purgeFolder("hdfs/path/to/folder", 10)
   * }}}
   *
@@ -156,7 +157,8 @@ object HdfsHelper extends Serializable {
 
       require(
         hdfs.isFile(fileToDelete),
-        "to delete a folder, prefer using the deleteFolder() method.")
+        "to delete a folder, prefer using the deleteFolder() method."
+      )
 
       hdfs.delete(fileToDelete, true)
     }
@@ -176,7 +178,8 @@ object HdfsHelper extends Serializable {
 
       require(
         !hdfs.isFile(folderToDelete),
-        "to delete a file, prefer using the deleteFile() method.")
+        "to delete a file, prefer using the deleteFile() method."
+      )
 
       hdfs.delete(folderToDelete, true)
     }
@@ -202,7 +205,8 @@ object HdfsHelper extends Serializable {
     if (hdfs.exists(fileToCheck))
       require(
         hdfs.isFile(fileToCheck),
-        "to check if a folder exists, prefer using the folderExists() method.")
+        "to check if a folder exists, prefer using the folderExists() method."
+      )
 
     hdfs.exists(fileToCheck)
   }
@@ -219,7 +223,8 @@ object HdfsHelper extends Serializable {
     if (hdfs.exists(folderToCheck))
       require(
         !hdfs.isFile(folderToCheck),
-        "to check if a file exists, prefer using the fileExists() method.")
+        "to check if a file exists, prefer using the fileExists() method."
+      )
 
     hdfs.exists(folderToCheck)
   }
@@ -246,7 +251,8 @@ object HdfsHelper extends Serializable {
     if (hdfs.exists(fileToRename))
       require(
         hdfs.isFile(fileToRename),
-        "to move a folder, prefer using the moveFolder() method.")
+        "to move a folder, prefer using the moveFolder() method."
+      )
 
     if (overwrite)
       hdfs.delete(renamedFile, true)
@@ -254,7 +260,8 @@ object HdfsHelper extends Serializable {
       require(
         !hdfs.exists(renamedFile),
         "overwrite option set to false, but a file already exists at target " +
-          "location " + newPath)
+          "location " + newPath
+      )
 
     // Before moving the file to its final destination, we check if the folder
     // where to put the file exists, and if not we create it:
@@ -286,7 +293,8 @@ object HdfsHelper extends Serializable {
     if (hdfs.exists(folderToRename))
       require(
         !hdfs.isFile(folderToRename),
-        "to move a file, prefer using the moveFile() method.")
+        "to move a file, prefer using the moveFile() method."
+      )
 
     if (overwrite)
       hdfs.delete(renamedFolder, true)
@@ -294,7 +302,8 @@ object HdfsHelper extends Serializable {
       require(
         !hdfs.exists(renamedFolder),
         "overwrite option set to false, but a folder already exists at target " +
-          "location " + newPath)
+          "location " + newPath
+      )
 
     // Before moving the folder to its final destination, we check if the folder
     // where to put the folder exists, and if not we create it:
@@ -739,7 +748,8 @@ object HdfsHelper extends Serializable {
 
     require(
       purgeAge >= 0,
-      "the purgeAge provided \"" + purgeAge.toString + "\" must be superior to 0.")
+      "the purgeAge provided \"" + purgeAge.toString + "\" must be superior to 0."
+    )
 
     hdfs
       .listStatus(new Path(folderPath))
diff --git a/src/main/scala/com/spark_helper/Monitor.scala b/src/main/scala/com/spark_helper/Monitor.scala
index f5eeeca..060787c 100644
--- a/src/main/scala/com/spark_helper/Monitor.scala
+++ b/src/main/scala/com/spark_helper/Monitor.scala
@@ -534,7 +534,8 @@ object Monitor {
         require(
           logDirectory.nonEmpty,
           "to save the report, please specify the log folder using " +
-            "Monitor.setLogFolder(\"hdfs/path/to/log/folder\")")
+            "Monitor.setLogFolder(\"hdfs/path/to/log/folder\")"
+        )
     }
   }
 
diff --git a/src/main/scala/com/spark_helper/SparkHelper.scala b/src/main/scala/com/spark_helper/SparkHelper.scala
index f87edf9..9d05d96 100644
--- a/src/main/scala/com/spark_helper/SparkHelper.scala
+++ b/src/main/scala/com/spark_helper/SparkHelper.scala
@@ -24,17 +24,37 @@ import scala.util.Random
   * A few examples:
   *
   * {{{
-  * // Same as sc.saveAsTextFile(path), but the result is a single file:
+  * import com.spark_helper.SparkHelper._
+  *
+  * // Same as rdd.saveAsTextFile("path"), but the result is a single file (while
+  * // keeping the processing distributed):
   * rdd.saveAsSingleTextFile("/my/output/file/path.txt")
-  * // Same as SparkContext.textFile, but instead of reading one record per
-  * // line, it reads records spread over several lines.
-  * // This way, xml, json, yml or any multi-line record file format can be used
-  * // with Spark:
-  * SparkHelper.textFileWithDelimiter("/my/input/folder/path", sparkContext, "---\n")
-  * // Same as SparkContext.textFile, but instead of returning an RDD of
-  * // records, it returns an RDD of tuples containing both the record and the
-  * // path of the file it comes from:
-  * SparkHelper.textFileWithFileName("folder", sparkContext)
+  * rdd.saveAsSingleTextFile("/my/output/file/path.txt", classOf[BZip2Codec])
+  *
+  * // Same as sc.textFile("path"), but instead of reading one record per line (by
+  * // splitting the input with \n), it splits the file in records based on a custom
+  * // delimiter. This way, xml, json, yml or any multi-line record file format can
+  * // be used with Spark:
+  * sc.textFile("/my/input/folder/path", "---\n") // for a yml file for instance
+  *
+  * // Equivalent to rdd.flatMap(identity) for RDDs of Seqs or Options:
+  * rdd.flatten
+  *
+  * // Equivalent to sc.textFile(), but for each line is tupled with its file path:
+  * sc.textFileWithFileName("/my/input/folder/path")
+  * // which produces:
+  * // RDD(("folder/file_1.txt", "record1fromfile1"), ("folder/file_1.txt", "record2fromfile1"),
+  * //    ("folder/file_2.txt", "record1fromfile2"), ...)
+  *
+  * // In the given folder, this generates one file per key in the given key/value
+  * // RDD. Within each file (named from the key) are all values for this key:
+  * rdd.saveAsTextFileByKey("/my/output/folder/path")
+  *
+  * // Concept mapper (the following example transforms RDD(1, 3, 2, 7, 8) into RDD(1, 3, 4, 7, 16)):
+  * rdd.partialMap { case a if a % 2 == 0 => 2 * a }
+  *
+  * // For when input files contain commas and textFile can't handle it:
+  * sc.textFile(Seq("path/hello,world.txt", "path/hello_world.txt"))
   * }}}
   *
   * Source <a href="https://github.com/xavierguihot/spark_helper/blob/master/src

From b88c19d2a34f646b598c533907f07899b667b638 Mon Sep 17 00:00:00 2001
From: Xavier GUIHOT <x.guihot@gmail.com>
Date: Sun, 17 Jun 2018 13:17:25 +0100
Subject: [PATCH 25/25] Bump version 2.0.0

---
 README.md | 8 +++-----
 build.sbt | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 50eb2bd..90015ab 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,6 @@
 ## Overview
 
 
-Version: 1.1.1
-
 API Scaladoc: [SparkHelper](http://xavierguihot.com/spark_helper/#com.spark_helper.SparkHelper$)
 
 This library contains a bunch of low-level basic methods for data processing
@@ -300,7 +298,7 @@ With sbt:
 ```scala
 resolvers += "jitpack" at "https://jitpack.io"
 
-libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "v1.1.1"
+libraryDependencies += "com.github.xavierguihot" % "spark_helper" % "2.0.0"
 ```
 
 With maven:
@@ -316,7 +314,7 @@ With maven:
 <dependency>
 	<groupId>com.github.xavierguihot</groupId>
 	<artifactId>spark_helper</artifactId>
-	<version>v1.1.1</version>
+	<version>2.0.0</version>
 </dependency>
 ```
 
@@ -330,7 +328,7 @@ allprojects {
 }
 
 dependencies {
-	compile 'com.github.xavierguihot:spark_helper:v1.1.1'
+	compile 'com.github.xavierguihot:spark_helper:2.0.0'
 }
 ```
 
diff --git a/build.sbt b/build.sbt
index bc2ebbc..15d0838 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,6 +1,6 @@
 name := "spark_helper"
 
-version := "1.1.1"
+version := "2.0.0"
 
 scalaVersion := "2.11.12"