Merge branch 'release/0.13.0'

.gitignore

@@ -32,3 +32,4 @@ tutorial/data/tmp3.tsv
 tutorial/data/jsonoutput0.tsv
 tutorial/data/avrooutput0.avro
 .scalding_repl
+scalding-hadoop-test/NOTICE

.travis.yml

@@ -1,12 +1,3 @@
-# Generate base tests:::
-# for s in 2.10.4 2.9.3; do
-#   for t in `ls -d scalding-*`; do
-# echo "    - scala: $s"
-# echo "      env: BUILD=\"base\" TEST_TARGET=\"$t\""
-# echo "      script: \"scripts/run_test.sh\""
-# echo ""
-# done
-# done
 language: scala
 jdk: oraclejdk7
 sudo: false
@@ -17,138 +8,94 @@ matrix:
   include:
 #BASE TESTS
     - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-args"
+      env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
       script: "scripts/run_test.sh"
 
-    - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-avro"
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-commons"
+      env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
+      script: "scripts/run_test.sh"
+
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-core"
       script: "scripts/run_test.sh"
 
-    - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-date"
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-core"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-hraven"
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-jdbc"
+      env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
       script: "scripts/run_test.sh"
 
-    - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-json"
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-parquet"
+      env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
       script: "scripts/run_test.sh"
 
-    - scala: 2.10.4
-      env: BUILD="base" TEST_TARGET="scalding-parquet-scrooge"
+    - scala: 2.11.4
+      env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-repl"
       script: "scripts/run_test.sh"
 
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-args"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-avro"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-commons"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-core"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-date"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-hraven"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-jdbc"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-json"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-parquet"
-      script: "scripts/run_test.sh"
-
-    - scala: 2.9.3
-      env: BUILD="base" TEST_TARGET="scalding-repl"
-      script: "scripts/run_test.sh"
-
     - scala: 2.10.4
       env: BUILD="test tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
       - "scripts/test_tutorials.sh"
-    - scala: 2.10.4
-      env: BUILD="test matrix tutorials"
+
+    - scala: 2.11.4
+      env: BUILD="test tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
-      - "scripts/test_matrix_tutorials.sh"
-    - scala: 2.10.4
-      env: BUILD="test repl tutorials"
-      script:
-      - "scripts/build_assembly_no_test.sh scalding-repl"
-      - "scripts/test_repl_tutorial.sh"
+      - "scripts/test_tutorials.sh"
+
     - scala: 2.10.4
-      env: BUILD="test typed tutorials"
-      script:
-      - "scripts/build_assembly_no_test.sh scalding-core"
-      - "scripts/test_typed_tutorials.sh"
-    - scala: 2.9.3
-      env: BUILD="test tutorials"
+      env: BUILD="test matrix tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
-      - "scripts/test_tutorials.sh"
-    - scala: 2.9.3
+      - "scripts/test_matrix_tutorials.sh"
+
+    - scala: 2.11.4
       env: BUILD="test matrix tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
       - "scripts/test_matrix_tutorials.sh"
-    - scala: 2.9.3
-      env: BUILD="test repl tutorials"
+
+    - scala: 2.10.4
+      env: BUILD="test repl and typed tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding-repl"
       - "scripts/test_repl_tutorial.sh"
-    - scala: 2.9.3
+      - "scripts/build_assembly_no_test.sh scalding-core"
+      - "scripts/test_typed_tutorials.sh"
+
+    - scala: 2.11.4
       env: BUILD="test typed tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding-core"
       - "scripts/test_typed_tutorials.sh"
 
 notifications:
   irc: "chat.freenode.net#scalding"
-

CHANGES.md

@@ -1,5 +1,35 @@
 # Scalding #
 
+### Version 0.13.0 ###
+* Covert LzoTextDelimited to Cascading scheme.: https://github.com/twitter/scalding/pull/1179
+* Make TraceUtil support versions of cascading older than 2.6: https://github.com/twitter/scalding/pull/1180
+* Add support for more LzoTextDeilmited parameters in LzoTraits.: https://github.com/twitter/scalding/pull/1178
+* Use latest algebird, bijection, chill, elephantbird, and scala 2.11.5: https://github.com/twitter/scalding/pull/1174
+* Cascading 2.6 tracing: https://github.com/twitter/scalding/pull/1156
+* use Cascading 2.6.1 and cascading-jdbc 2.6.0: https://github.com/twitter/scalding/pull/1110
+* add reducer option to LookupJoin: https://github.com/twitter/scalding/pull/1160
+* Add dump to ValuePipe in the REPL: https://github.com/twitter/scalding/pull/1157
+* Ianoc/type descriptor: https://github.com/twitter/scalding/pull/1147
+* Refactor around the macro definitions into 3 files. Both converter and setter support Options: https://github.com/twitter/scalding/pull/1145
+* Fix a few random typos: https://github.com/twitter/scalding/pull/1144
+* Fix two issues found by static analysis: https://github.com/twitter/scalding/pull/1143
+* Add implicit helpers for numeric arguments: https://github.com/twitter/scalding/pull/1138
+* Add a fields macro: https://github.com/twitter/scalding/pull/1132
+* Ianoc/case class tuple converters: https://github.com/twitter/scalding/pull/1131
+* Some minor changes, cleanup pulled from jco's macro branch: https://github.com/twitter/scalding/pull/1130
+* Adds a typedjson source: https://github.com/twitter/scalding/pull/1129
+* Pulls all external 3rdparty versions up to the top of the build file: https://github.com/twitter/scalding/pull/1128
+* remove transitive pig and elephantbird dependencies for parquet-cascading: https://github.com/twitter/scalding/pull/1127
+* Some minor clean up in the build file: https://github.com/twitter/scalding/pull/1123
+* Ianoc/scalding 210: https://github.com/twitter/scalding/pull/1116
+* Decrease test count: https://github.com/twitter/scalding/pull/1117
+* Removes scala 2.9.3: https://github.com/twitter/scalding/pull/1106
+* Fix some typos in TypedPipe docs, expand flatMap docs: https://github.com/twitter/scalding/pull/1115
+* Implicit execution context / easier switching between modes: https://github.com/twitter/scalding/pull/1113
+* Add more documentation to TypedPipe: https://github.com/twitter/scalding/pull/1111
+* Update the README: https://github.com/twitter/scalding/pull/1114
+* Fixed comment in LookupJoin.scala: https://github.com/twitter/scalding/pull/1108
+
 ### Version 0.12.0 ###
 * Fix long compile time for MultiJoin helpers: https://github.com/twitter/scalding/pull/1109
 * Allows reducer estimation to operate on all hfs taps: https://github.com/twitter/scalding/pull/1080

README.md

@@ -4,7 +4,7 @@ Scalding is a Scala library that makes it easy to specify Hadoop MapReduce jobs.
 
 ![Scalding Logo](https://raw.github.com/twitter/scalding/develop/logo/scalding.png)
 
-Current version: `0.12.0`
+Current version: `0.13.0`
 
 ## Word Count
 
@@ -15,11 +15,12 @@ package com.twitter.scalding.examples
 
 import com.twitter.scalding._
 
-class WordCountJob(args : Args) extends Job(args) {
-  TextLine( args("input") )
-    .flatMap('line -> 'word) { line : String => tokenize(line) }
-    .groupBy('word) { _.size }
-    .write( Tsv( args("output") ) )
+class WordCountJob(args: Args) extends Job(args) {
+  TypedPipe.from(TextLine(args("input")))
+    .flatMap { line => tokenize(line) }
+    .groupBy { word => word } // use each word for a key
+    .size // in each group, get the size
+    .write(TypedTsv[(String, Long)](args("output")))
 
   // Split a piece of text into individual words.
   def tokenize(text : String) : Array[String] = {
@@ -36,17 +37,22 @@ You can find more example code under [examples/](https://github.com/twitter/scal
 ## Documentation and Getting Started
 
 * [**Getting Started**](https://github.com/twitter/scalding/wiki/Getting-Started) page on the [Scalding Wiki](https://github.com/twitter/scalding/wiki)
+* [**REPL in Wonderland**](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) a hands-on tour of the
+  scalding REPL requiring only git and java installed.
 * [**Runnable tutorials**](https://github.com/twitter/scalding/tree/master/tutorial) in the source.
 * The API Reference, including many example Scalding snippets:
-  * [Fields-based API Reference](https://github.com/twitter/scalding/wiki/Fields-based-API-Reference)
   * [Type-safe API Reference](https://github.com/twitter/scalding/wiki/Type-safe-api-reference)
+  * [Fields-based API Reference](https://github.com/twitter/scalding/wiki/Fields-based-API-Reference)
 * [Scalding Scaladocs](http://twitter.github.com/scalding) provide details beyond the API References
 * The Matrix Library provides a way of working with key-attribute-value scalding pipes:
   * The [Introduction to Matrix Library](https://github.com/twitter/scalding/wiki/Introduction-to-Matrix-Library) contains an overview and a "getting started" example
   * The [Matrix API Reference](https://github.com/twitter/scalding/wiki/Matrix-API-Reference) contains the Matrix Library API reference with examples
 
 Please feel free to use the beautiful [Scalding logo](https://drive.google.com/folderview?id=0B3i3pDi3yVgNbm9pMUdDcHFKVEk&usp=sharing) artwork anywhere.
 
+## Code of Conduct
+This, and all github.com/twitter projects, are under the [Twitter Open Source Code of Conduct](https://engineering.twitter.com/opensource/code-of-conduct). Additionally, see the [Typelevel Code of Conduct](http://typelevel.org/conduct) for specific examples of harassing behavior that are not tolerated.
+
 ## Building
 There is a script (called sbt) in the root that loads the correct sbt version to build:
 
@@ -63,25 +69,21 @@ Please refer to [FAQ page](https://github.com/twitter/scalding/wiki/Frequently-a
 We use [Travis CI](http://travis-ci.org/) to verify the build:
 [![Build Status](https://secure.travis-ci.org/twitter/scalding.png)](http://travis-ci.org/twitter/scalding)
 
+We use [Coveralls](https://coveralls.io/r/twitter/scalding) for code coverage results:
+[![Coverage Status](https://coveralls.io/repos/twitter/scalding/badge.png?branch=develop)](https://coveralls.io/r/twitter/scalding?branch=develop)
+
 Scalding modules are available from maven central.
 
-The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.11.0`.
+The current groupid and version for all modules is, respectively, `"com.twitter"` and  `0.12.0`.
 
 Current published artifacts are
 
-* `scalding-core_2.9.3`
 * `scalding-core_2.10`
-* `scalding-args_2.9.3`
 * `scalding-args_2.10`
-* `scalding-date_2.9.3`
 * `scalding-date_2.10`
-* `scalding-commons_2.9.3`
 * `scalding-commons_2.10`
-* `scalding-avro_2.9.3`
 * `scalding-avro_2.10`
-* `scalding-parquet_2.9.3`
 * `scalding-parquet_2.10`
-* `scalding-repl_2.9.3`
 * `scalding-repl_2.10`