From 6447bdb91edabc9ce2316fca93d4d73419eb66bd Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 11 Feb 2016 12:11:05 -0800 Subject: [PATCH 1/8] add scalding-parquet-cascading --- build.sbt | 20 ++++++++++++++----- .../cascading}/ParquetValueScheme.java | 0 .../cascading}/thrift/ParquetTBaseScheme.java | 0 .../tuple/ParquetTupleConverter.java | 0 .../cascading}/tuple/ParquetTupleScheme.java | 0 .../cascading}/tuple/SchemaIntersection.java | 0 .../cascading}/tuple/TupleReadSupport.java | 0 .../tuple/TupleRecordMaterializer.java | 0 .../cascading}/tuple/TupleWriteSupport.java | 0 .../thrift/Parquet346TBaseScheme.scala | 0 .../thrift/TestParquetTBaseScheme.java | 0 .../tuple/TestParquetTupleScheme.java | 0 .../src/test/resources/names.txt | 0 13 files changed, 15 insertions(+), 5 deletions(-) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/ParquetValueScheme.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/thrift/ParquetTBaseScheme.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/ParquetTupleConverter.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/ParquetTupleScheme.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/SchemaIntersection.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/TupleReadSupport.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/TupleRecordMaterializer.java (100%) rename {scalding-parquet/src/main/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading}/tuple/TupleWriteSupport.java (100%) rename {scalding-parquet/src/main/scala/com/twitter/scalding/parquet => scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading}/thrift/Parquet346TBaseScheme.scala (100%) rename {scalding-parquet/src/test/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading}/thrift/TestParquetTBaseScheme.java (100%) rename {scalding-parquet/src/test/java/com/twitter/scalding/parquet => scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading}/tuple/TestParquetTupleScheme.java (100%) rename {scalding-parquet => scalding-parquet-cascading}/src/test/resources/names.txt (100%) diff --git a/build.sbt b/build.sbt index efaf34e8e2..8524e1fe7f 100644 --- a/build.sbt +++ b/build.sbt @@ -384,8 +384,8 @@ lazy val scaldingParquetFixtures = module("parquet-fixtures") ) ) -lazy val scaldingParquet = module("parquet").settings( - libraryDependencies <++= (scalaVersion) { scalaVersion => Seq( +lazy val scaldingParquetCascading = module("parquet-cascading").settings( + libraryDependencies ++= Seq( "org.apache.parquet" % "parquet-column" % parquetVersion, "org.apache.parquet" % "parquet-hadoop" % parquetVersion, "org.apache.parquet" % "parquet-thrift" % parquetVersion @@ -394,15 +394,25 @@ lazy val scaldingParquet = module("parquet").settings( exclude("com.twitter.elephantbird", "elephant-bird-pig") exclude("com.twitter.elephantbird", "elephant-bird-core"), "org.apache.thrift" % "libthrift" % "0.7.0", + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "cascading" % "cascading-core" % cascadingVersion, + "cascading" % "cascading-hadoop" % cascadingVersion, + "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion % "test" + ) +).dependsOn(scaldingParquetFixtures % "test->test") + +lazy val scaldingParquet = module("parquet").settings( + libraryDependencies <++= (scalaVersion) { scalaVersion => Seq( + "org.apache.parquet" % "parquet-column" % parquetVersion, + "org.apache.parquet" % "parquet-hadoop" % parquetVersion, "org.slf4j" % "slf4j-api" % slf4jVersion, "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", "org.scala-lang" % "scala-reflect" % scalaVersion, "com.twitter" %% "bijection-macros" % bijectionVersion, - "com.twitter" %% "chill-bijection" % chillVersion, - "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion % "test" + "com.twitter" %% "chill-bijection" % chillVersion ) ++ (if(isScala210x(scalaVersion)) Seq("org.scalamacros" %% "quasiquotes" % quasiquotesVersion) else Seq()) }, addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full)) - .dependsOn(scaldingCore, scaldingHadoopTest % "test", scaldingParquetFixtures % "test->test") + .dependsOn(scaldingCore, scaldingParquetCascading, scaldingHadoopTest % "test") diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/ParquetValueScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/ParquetValueScheme.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/thrift/ParquetTBaseScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/thrift/ParquetTBaseScheme.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/ParquetTupleConverter.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/ParquetTupleConverter.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/ParquetTupleScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/ParquetTupleScheme.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/SchemaIntersection.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/SchemaIntersection.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleReadSupport.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleReadSupport.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleRecordMaterializer.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleRecordMaterializer.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java diff --git a/scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleWriteSupport.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java similarity index 100% rename from scalding-parquet/src/main/java/com/twitter/scalding/parquet/tuple/TupleWriteSupport.java rename to scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala b/scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala similarity index 100% rename from scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/Parquet346TBaseScheme.scala rename to scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala diff --git a/scalding-parquet/src/test/java/com/twitter/scalding/parquet/thrift/TestParquetTBaseScheme.java b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java similarity index 100% rename from scalding-parquet/src/test/java/com/twitter/scalding/parquet/thrift/TestParquetTBaseScheme.java rename to scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java diff --git a/scalding-parquet/src/test/java/com/twitter/scalding/parquet/tuple/TestParquetTupleScheme.java b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java similarity index 100% rename from scalding-parquet/src/test/java/com/twitter/scalding/parquet/tuple/TestParquetTupleScheme.java rename to scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java diff --git a/scalding-parquet/src/test/resources/names.txt b/scalding-parquet-cascading/src/test/resources/names.txt similarity index 100% rename from scalding-parquet/src/test/resources/names.txt rename to scalding-parquet-cascading/src/test/resources/names.txt From 2f2feb9c1396dd458ef8a9ddd21c0b31dcc0105a Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 11 Feb 2016 12:48:59 -0800 Subject: [PATCH 2/8] add scalding-parquet-scrooge-cascading --- build.sbt | 26 ++++++++++++++----- .../cascading}/ParquetScroogeScheme.java | 0 .../cascading}/ScroogeReadSupport.java | 0 .../cascading}/ScroogeRecordConverter.java | 0 .../ScroogeSchemaConversionException.java | 0 .../cascading}/ScroogeStructConverter.java | 0 .../cascading}/ScroogeWriteSupport.java | 0 .../cascading}/Parquet346ScroogeScheme.scala | 0 .../cascading}/ParquetScroogeSchemeTest.java | 0 .../src/test/resources/names.txt | 0 10 files changed, 19 insertions(+), 7 deletions(-) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ParquetScroogeScheme.java (100%) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeReadSupport.java (100%) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeRecordConverter.java (100%) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeSchemaConversionException.java (100%) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeStructConverter.java (100%) rename {scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeWriteSupport.java (100%) rename {scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading}/Parquet346ScroogeScheme.scala (100%) rename {scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading}/ParquetScroogeSchemeTest.java (100%) rename {scalding-parquet-scrooge => scalding-parquet-scrooge-cascading}/src/test/resources/names.txt (100%) diff --git a/build.sbt b/build.sbt index 8524e1fe7f..5d1e33d5c9 100644 --- a/build.sbt +++ b/build.sbt @@ -43,6 +43,7 @@ val scroogeVersion = "3.20.0" val slf4jVersion = "1.6.6" val thriftVersion = "0.5.0" val junitVersion = "4.10" +val junitInterfaceVersion = "0.11" val printDependencyClasspath = taskKey[Unit]("Prints location of the dependencies") @@ -64,7 +65,7 @@ val sharedSettings = Project.defaultSettings ++ assemblySettings ++ scalariformS "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test", - "com.novocode" % "junit-interface" % "0.10" % "test" + "com.novocode" % "junit-interface" % junitInterfaceVersion % "test" ), resolvers ++= Seq( @@ -414,8 +415,6 @@ lazy val scaldingParquet = module("parquet").settings( }, addCompilerPlugin("org.scalamacros" % "paradise" % paradiseVersion cross CrossVersion.full)) .dependsOn(scaldingCore, scaldingParquetCascading, scaldingHadoopTest % "test") - - lazy val scaldingParquetScroogeFixtures = module("parquet-scrooge-fixtures") .settings(ScroogeSBT.newSettings:_*) .settings( @@ -442,10 +441,9 @@ lazy val scaldingParquetScroogeFixtures = module("parquet-scrooge-fixtures") ) ) -lazy val scaldingParquetScrooge = module("parquet-scrooge") +lazy val scaldingParquetScroogeCascading = module("parquet-scrooge-cascading") .settings( libraryDependencies ++= Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion, // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions "org.apache.parquet" % "parquet-thrift" % parquetVersion % "test" classifier "tests" exclude("org.apache.parquet", "parquet-pig") @@ -453,11 +451,25 @@ lazy val scaldingParquetScrooge = module("parquet-scrooge") exclude("com.twitter.elephantbird", "elephant-bird-core"), "com.twitter" %% "scrooge-serializer" % scroogeVersion, "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", - "com.novocode" % "junit-interface" % "0.11" % "test", "junit" % "junit" % junitVersion % "test" + ) +).dependsOn(scaldingParquetCascading % "compile->compile;test->test", scaldingParquetScroogeFixtures % "test->test") +lazy val scaldingParquetScrooge = module("parquet-scrooge") + .settings( + libraryDependencies ++= Seq( + // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions + "org.apache.parquet" % "parquet-thrift" % parquetVersion % "test" classifier "tests" + exclude("org.apache.parquet", "parquet-pig") + exclude("com.twitter.elephantbird", "elephant-bird-pig") + exclude("com.twitter.elephantbird", "elephant-bird-core"), + "com.twitter" %% "scrooge-serializer" % scroogeVersion, + "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", + "com.twitter.elephantbird" % "elephant-bird-core" % elephantbirdVersion % "test", + "com.novocode" % "junit-interface" % junitInterfaceVersion % "test", + "junit" % "junit" % junitVersion % "test" ) -).dependsOn(scaldingCore, scaldingParquet % "compile->compile;test->test", scaldingParquetScroogeFixtures % "test->test") +).dependsOn(scaldingCore, scaldingParquetScroogeCascading, scaldingParquet % "compile->compile;test->test", scaldingParquetScroogeFixtures % "test->test") lazy val scaldingHRaven = module("hraven").settings( libraryDependencies ++= Seq( diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeScheme.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeScheme.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeReadSupport.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeReadSupport.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeRecordConverter.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeRecordConverter.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeSchemaConversionException.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeSchemaConversionException.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeStructConverter.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeStructConverter.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeWriteSupport.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java similarity index 100% rename from scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ScroogeWriteSupport.java rename to scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java diff --git a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala b/scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala similarity index 100% rename from scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/Parquet346ScroogeScheme.scala rename to scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala diff --git a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeSchemeTest.java b/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java similarity index 100% rename from scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeSchemeTest.java rename to scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java diff --git a/scalding-parquet-scrooge/src/test/resources/names.txt b/scalding-parquet-scrooge-cascading/src/test/resources/names.txt similarity index 100% rename from scalding-parquet-scrooge/src/test/resources/names.txt rename to scalding-parquet-scrooge-cascading/src/test/resources/names.txt From 119f87fc152940ae46d30240126f37a5a64017b1 Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 11 Feb 2016 14:53:03 -0800 Subject: [PATCH 3/8] update namespace, related imports --- .../scalding/parquet/cascading/ParquetValueScheme.java | 2 +- .../parquet/cascading/thrift/ParquetTBaseScheme.java | 4 ++-- .../parquet/cascading/tuple/ParquetTupleConverter.java | 2 +- .../parquet/cascading/tuple/ParquetTupleScheme.java | 2 +- .../parquet/cascading/tuple/SchemaIntersection.java | 2 +- .../parquet/cascading/tuple/TupleReadSupport.java | 2 +- .../cascading/tuple/TupleRecordMaterializer.java | 2 +- .../parquet/cascading/tuple/TupleWriteSupport.java | 2 +- .../cascading/thrift/Parquet346TBaseScheme.scala | 4 ++-- .../cascading/thrift/TestParquetTBaseScheme.java | 2 +- .../cascading/tuple/TestParquetTupleScheme.java | 2 +- .../scrooge/cascading/ParquetScroogeScheme.java | 4 ++-- .../parquet/scrooge/cascading/ScroogeReadSupport.java | 2 +- .../scrooge/cascading/ScroogeRecordConverter.java | 2 +- .../cascading/ScroogeSchemaConversionException.java | 2 +- .../scrooge/cascading/ScroogeStructConverter.java | 2 +- .../parquet/scrooge/cascading/ScroogeWriteSupport.java | 2 +- .../scrooge/cascading/Parquet346ScroogeScheme.scala | 6 +++--- .../scrooge/cascading/ParquetScroogeSchemeTest.java | 4 ++-- .../scrooge/cascading}/ScroogeStructConverterTest.java | 2 +- .../parquet/scrooge/ParquetScroogeOutputFormat.java | 1 + .../com/twitter/scalding/parquet/scrooge/package.scala | 6 ++++++ .../scala/com/twitter/scalding/parquet/package.scala | 10 ++++++++++ .../com/twitter/scalding/parquet/thrift/package.scala | 6 ++++++ .../com/twitter/scalding/parquet/tuple/package.scala | 6 ++++++ .../twitter/scalding/parquet/ParquetSourcesTests.scala | 2 +- 26 files changed, 56 insertions(+), 27 deletions(-) rename {scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge => scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading}/ScroogeStructConverterTest.java (99%) create mode 100644 scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/package.scala create mode 100644 scalding-parquet/src/main/scala/com/twitter/scalding/parquet/package.scala create mode 100644 scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/package.scala create mode 100644 scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/package.scala diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java index 2d71c44896..6d19286641 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/ParquetValueScheme.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet; +package com.twitter.scalding.parquet.cascading; import java.io.IOException; import java.io.Serializable; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java index d62596b98d..3d80483669 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/thrift/ParquetTBaseScheme.java @@ -1,6 +1,6 @@ -package com.twitter.scalding.parquet.thrift; +package com.twitter.scalding.parquet.cascading.thrift; -import com.twitter.scalding.parquet.ParquetValueScheme; +import com.twitter.scalding.parquet.cascading.ParquetValueScheme; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java index 4f313d7392..8fb922560c 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleConverter.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import cascading.tuple.Tuple; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java index 203f3cd67c..deeb94116c 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/ParquetTupleScheme.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import java.io.IOException; import java.util.List; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java index 4afee64f8a..c1793f0b66 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/SchemaIntersection.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.Type; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java index e3dd32dc2f..17c763274c 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleReadSupport.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import java.util.Map; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java index ba53b20722..67a2bbd312 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleRecordMaterializer.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import cascading.tuple.Tuple; diff --git a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java index 69b0cc382c..e51da4eb0d 100644 --- a/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java +++ b/scalding-parquet-cascading/src/main/java/com/twitter/scalding/parquet/cascading/tuple/TupleWriteSupport.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import cascading.tuple.TupleEntry; import java.util.HashMap; diff --git a/scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala b/scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala index d71623dcd4..1110b78119 100644 --- a/scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala +++ b/scalding-parquet-cascading/src/main/scala/com/twitter/scalding/parquet/cascading/thrift/Parquet346TBaseScheme.scala @@ -1,6 +1,6 @@ -package com.twitter.scalding.parquet.thrift +package com.twitter.scalding.parquet.cascading.thrift -import com.twitter.scalding.parquet.ParquetValueScheme +import com.twitter.scalding.parquet.cascading.ParquetValueScheme import cascading.flow.FlowProcess import cascading.tap.Tap diff --git a/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java index 5d4a2fb898..1c2a5ec29a 100644 --- a/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java +++ b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/thrift/TestParquetTBaseScheme.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.thrift; +package com.twitter.scalding.parquet.cascading.thrift; import com.twitter.scalding.parquet.thrift_java.test.Name; diff --git a/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java index b6300a7857..7a55ad59f7 100644 --- a/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java +++ b/scalding-parquet-cascading/src/test/java/com/twitter/scalding/parquet/cascading/tuple/TestParquetTupleScheme.java @@ -1,4 +1,4 @@ -package com.twitter.scalding.parquet.tuple; +package com.twitter.scalding.parquet.cascading.tuple; import com.twitter.scalding.parquet.thrift_java.test.Name; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java index cff692b347..7a24c2f9f6 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeScheme.java @@ -16,13 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.RecordReader; -import com.twitter.scalding.parquet.ParquetValueScheme; +import com.twitter.scalding.parquet.cascading.ParquetValueScheme; import com.twitter.scrooge.ThriftStruct; import cascading.flow.FlowProcess; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java index 146912395a..91633c9e04 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeReadSupport.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import org.apache.parquet.hadoop.thrift.ThriftReadSupport; import org.apache.parquet.schema.MessageType; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java index 20f1e31fe4..245bfc1dd7 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeRecordConverter.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import org.apache.thrift.TException; import org.apache.thrift.protocol.TProtocol; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java index 6c6d5cb630..85897ee05f 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeSchemaConversionException.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import org.apache.parquet.ParquetRuntimeException; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java index 16f7851bca..1f1d510c3c 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverter.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; diff --git a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java index 5d9f3ffb72..6111479b8a 100644 --- a/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java +++ b/scalding-parquet-scrooge-cascading/src/main/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeWriteSupport.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import com.twitter.scrooge.ThriftStruct; diff --git a/scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala b/scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala index afc15ebc6b..9189f4214d 100644 --- a/scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala +++ b/scalding-parquet-scrooge-cascading/src/main/scala/com/twitter/scalding/parquet/scrooge/cascading/Parquet346ScroogeScheme.scala @@ -1,9 +1,9 @@ -package com.twitter.scalding.parquet.scrooge +package com.twitter.scalding.parquet.cascading.scrooge import cascading.flow.FlowProcess import cascading.tap.Tap -import com.twitter.scalding.parquet.ParquetValueScheme -import com.twitter.scalding.parquet.thrift.Parquet346StructTypeRepairer +import com.twitter.scalding.parquet.cascading.ParquetValueScheme +import com.twitter.scalding.parquet.cascading.thrift.Parquet346StructTypeRepairer import com.twitter.scrooge.{ ThriftStruct, ThriftStructCodec } import org.apache.hadoop.mapred.{ JobConf, OutputCollector, RecordReader } import org.apache.parquet.hadoop.thrift.ThriftReadSupport diff --git a/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java b/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java index bd626832ea..a0c6aecb5d 100644 --- a/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java +++ b/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ParquetScroogeSchemeTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import cascading.flow.Flow; import cascading.flow.FlowProcess; @@ -47,7 +47,7 @@ import org.junit.Test; import org.apache.parquet.hadoop.thrift.ThriftToParquetFileWriter; import org.apache.parquet.hadoop.util.ContextUtil; -import com.twitter.scalding.parquet.ParquetValueScheme.Config; +import com.twitter.scalding.parquet.cascading.ParquetValueScheme.Config; import com.twitter.scalding.parquet.scrooge.thrift_scala.test.TestPersonWithAllInformation; import com.twitter.scalding.parquet.scrooge.thrift_java.test.Address; import com.twitter.scalding.parquet.scrooge.thrift_java.test.Phone; diff --git a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeStructConverterTest.java b/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverterTest.java similarity index 99% rename from scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeStructConverterTest.java rename to scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverterTest.java index c8dec25947..0118e52888 100644 --- a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeStructConverterTest.java +++ b/scalding-parquet-scrooge-cascading/src/test/java/com/twitter/scalding/parquet/scrooge/cascading/ScroogeStructConverterTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.twitter.scalding.parquet.scrooge; +package com.twitter.scalding.parquet.cascading.scrooge; import org.apache.thrift.TBase; import org.junit.Test; diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeOutputFormat.java b/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeOutputFormat.java index daa2f69545..13ea61813f 100644 --- a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeOutputFormat.java +++ b/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeOutputFormat.java @@ -15,6 +15,7 @@ */ package com.twitter.scalding.parquet.scrooge; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeWriteSupport; import com.twitter.scrooge.ThriftStruct; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.hadoop.ParquetOutputFormat; diff --git a/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/package.scala b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/package.scala new file mode 100644 index 0000000000..3305ad179a --- /dev/null +++ b/scalding-parquet-scrooge/src/main/scala/com/twitter/scalding/parquet/scrooge/package.scala @@ -0,0 +1,6 @@ +package com.twitter.scalding.parquet + +package object scrooge { + type Parquet346ScroogeScheme[T <: com.twitter.scrooge.ThriftStruct] = com.twitter.scalding.parquet.cascading.scrooge.Parquet346ScroogeScheme[T] +} + diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/package.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/package.scala new file mode 100644 index 0000000000..ddc48e0e43 --- /dev/null +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/package.scala @@ -0,0 +1,10 @@ +package com.twitter.scalding + +package object parquet { + type ParquetValueScheme[T] = com.twitter.scalding.parquet.cascading.ParquetValueScheme[T] + + object ParquetValueScheme { + type Config[T] = com.twitter.scalding.parquet.cascading.ParquetValueScheme.Config[T] + } +} + diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/package.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/package.scala new file mode 100644 index 0000000000..a6edade5b0 --- /dev/null +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/thrift/package.scala @@ -0,0 +1,6 @@ +package com.twitter.scalding.parquet + +package object thrift { + type Parquet346TBaseScheme[T <: org.apache.thrift.TBase[_, _]] = com.twitter.scalding.parquet.cascading.thrift.Parquet346TBaseScheme[T] +} + diff --git a/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/package.scala b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/package.scala new file mode 100644 index 0000000000..79bb10f7ca --- /dev/null +++ b/scalding-parquet/src/main/scala/com/twitter/scalding/parquet/tuple/package.scala @@ -0,0 +1,6 @@ +package com.twitter.scalding.parquet + +package object tuple { + type ParquetTupleScheme = com.twitter.scalding.parquet.cascading.tuple.ParquetTupleScheme +} + diff --git a/scalding-parquet/src/test/scala/com/twitter/scalding/parquet/ParquetSourcesTests.scala b/scalding-parquet/src/test/scala/com/twitter/scalding/parquet/ParquetSourcesTests.scala index 82689a35f5..3dd25d14c8 100644 --- a/scalding-parquet/src/test/scala/com/twitter/scalding/parquet/ParquetSourcesTests.scala +++ b/scalding-parquet/src/test/scala/com/twitter/scalding/parquet/ParquetSourcesTests.scala @@ -1,6 +1,6 @@ package com.twitter.scalding.parquet -import cascading.tuple.Fields +import _root_.cascading.tuple.Fields import com.twitter.scalding.parquet.thrift.{ DailySuffixParquetThrift, FixedPathParquetThrift, HourlySuffixParquetThrift } import com.twitter.scalding.parquet.tuple.{ DailySuffixParquetTuple, FixedPathParquetTuple, HourlySuffixParquetTuple } import com.twitter.scalding.{ DateRange, RichDate, Source } From 96607da9733503d8653baea282b136aa964ebe7c Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 11 Feb 2016 15:09:45 -0800 Subject: [PATCH 4/8] update travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a9a5a1ec11..cbfd84ce51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,11 +23,11 @@ matrix: script: "scripts/run_test.sh" - scala: 2.10.6 - env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-scrooge" + env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-cascading scalding-parquet-scrooge scalding-parquet-scrooge-cascading" script: "scripts/run_test.sh" - scala: 2.11.7 - env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-scrooge" + env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-cascading scalding-parquet-scrooge scalding-parquet-scrooge-cascading" script: "scripts/run_test.sh" - scala: 2.10.6 From 443929f7195e62dc54bee1bf1b4e643c264a30c3 Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Thu, 11 Feb 2016 15:53:49 -0800 Subject: [PATCH 5/8] fix missing import --- .../scalding/parquet/scrooge/ParquetScroogeInputFormat.java | 1 + 1 file changed, 1 insertion(+) diff --git a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeInputFormat.java b/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeInputFormat.java index e58a8f731b..4d31a4e447 100644 --- a/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeInputFormat.java +++ b/scalding-parquet-scrooge/src/main/java/com/twitter/scalding/parquet/scrooge/ParquetScroogeInputFormat.java @@ -18,6 +18,7 @@ */ package com.twitter.scalding.parquet.scrooge; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeReadSupport; import org.apache.parquet.hadoop.thrift.ParquetThriftInputFormat; /** From 80796332d44387ba98143d0ddef78d83c8813242 Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Fri, 12 Feb 2016 08:51:39 -0800 Subject: [PATCH 6/8] fix another missing import --- .../com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java index 0686e79405..80de2b221e 100644 --- a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java +++ b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java @@ -29,6 +29,7 @@ import org.junit.rules.TemporaryFolder; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.ParquetWriter; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeWriteSupport; import com.twitter.scalding.parquet.scrooge.thrift_scala.test.StringAndBinary; import org.apache.parquet.thrift.ThriftParquetReader; From 46bb22f9930198a61b04b73a01ae99fcbd35e6dc Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Fri, 12 Feb 2016 10:09:53 -0800 Subject: [PATCH 7/8] fix missing imports --- .../com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java | 2 ++ .../scalding/parquet/scrooge/TestCorruptScroogeRecords.java | 1 + 2 files changed, 3 insertions(+) diff --git a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java index 80de2b221e..59c6598283 100644 --- a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java +++ b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/ScroogeBinaryTest.java @@ -29,6 +29,8 @@ import org.junit.rules.TemporaryFolder; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.ParquetWriter; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeReadSupport; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeRecordConverter; import com.twitter.scalding.parquet.cascading.scrooge.ScroogeWriteSupport; import com.twitter.scalding.parquet.scrooge.thrift_scala.test.StringAndBinary; import org.apache.parquet.thrift.ThriftParquetReader; diff --git a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/TestCorruptScroogeRecords.java b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/TestCorruptScroogeRecords.java index f5439e84be..0e9c30cd8f 100644 --- a/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/TestCorruptScroogeRecords.java +++ b/scalding-parquet-scrooge/src/test/java/com/twitter/scalding/parquet/scrooge/TestCorruptScroogeRecords.java @@ -33,6 +33,7 @@ import org.apache.parquet.hadoop.thrift.TestCorruptThriftRecords; import org.apache.parquet.hadoop.thrift.ThriftReadSupport; +import com.twitter.scalding.parquet.cascading.scrooge.ScroogeRecordConverter; import com.twitter.scalding.parquet.scrooge.thrift_scala.test.StructWithUnionV2; import com.twitter.scalding.parquet.scrooge.thrift_scala.test.StructWithUnionV2$; From 3d1e76a776bfae137bfda69ca4f5ddbc813fffff Mon Sep 17 00:00:00 2001 From: Ruban Monu Date: Fri, 12 Feb 2016 11:16:38 -0800 Subject: [PATCH 8/8] review comments build file --- build.sbt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 5d1e33d5c9..1d3b31e86d 100644 --- a/build.sbt +++ b/build.sbt @@ -385,6 +385,7 @@ lazy val scaldingParquetFixtures = module("parquet-fixtures") ) ) +// separate target that only depends on parquet, thrift, eb and cascading. Not scalding. lazy val scaldingParquetCascading = module("parquet-cascading").settings( libraryDependencies ++= Seq( "org.apache.parquet" % "parquet-column" % parquetVersion, @@ -394,7 +395,7 @@ lazy val scaldingParquetCascading = module("parquet-cascading").settings( exclude("org.apache.parquet", "parquet-pig") exclude("com.twitter.elephantbird", "elephant-bird-pig") exclude("com.twitter.elephantbird", "elephant-bird-core"), - "org.apache.thrift" % "libthrift" % "0.7.0", + "org.apache.thrift" % "libthrift" % thriftVersion, "org.apache.hadoop" % "hadoop-client" % hadoopVersion % "provided", "cascading" % "cascading-core" % cascadingVersion, "cascading" % "cascading-hadoop" % cascadingVersion, @@ -441,10 +442,12 @@ lazy val scaldingParquetScroogeFixtures = module("parquet-scrooge-fixtures") ) ) +// separate target that only depends on parquet, scrooge, eb and cascading. Not scalding. lazy val scaldingParquetScroogeCascading = module("parquet-scrooge-cascading") .settings( libraryDependencies ++= Seq( // see https://issues.apache.org/jira/browse/PARQUET-143 for exclusions + "cascading" % "cascading-core" % cascadingVersion, "org.apache.parquet" % "parquet-thrift" % parquetVersion % "test" classifier "tests" exclude("org.apache.parquet", "parquet-pig") exclude("com.twitter.elephantbird", "elephant-bird-pig")