Skip to content

Commit

Permalink
Merge branch 'release/0.15.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
Katya Gonina committed May 21, 2015
2 parents 83dba38 + df04598 commit afbde5a
Show file tree
Hide file tree
Showing 182 changed files with 10,674 additions and 1,075 deletions.
62 changes: 41 additions & 21 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,95 +7,115 @@ script:
matrix:
include:
#BASE TESTS
- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-core"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-core"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-macros"
script: "scripts/run_test.sh"

- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-macros"
script: "scripts/run_test.sh"

# not committed yet
# - scala: 2.10.5
# env: BUILD="base" TEST_TARGET="scalding-commons-macros"
# script: "scripts/run_test.sh"

# - scala: 2.11.5
# env: BUILD="base" TEST_TARGET="scalding-commons-macros"
# script: "scripts/run_test.sh"

- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
script: "scripts/run_test.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="base" TEST_TARGET="scalding-repl"
script: "scripts/run_test.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="test tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding"
- "scripts/test_tutorials.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="test tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding"
- "scripts/test_tutorials.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="test matrix tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding"
- "scripts/test_matrix_tutorials.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="test matrix tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding"
- "scripts/test_matrix_tutorials.sh"

- scala: 2.10.4
- scala: 2.10.5
env: BUILD="test repl and typed tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding-repl"
- "scripts/test_repl_tutorial.sh"
- "scripts/build_assembly_no_test.sh scalding-core"
- "scripts/test_typed_tutorials.sh"

- scala: 2.11.4
- scala: 2.11.5
env: BUILD="test typed tutorials"
script:
- "scripts/build_assembly_no_test.sh scalding-core"
- "scripts/test_typed_tutorials.sh"

notifications:
irc: "chat.freenode.net#scalding"
- scala: 2.10.5
env: BUILD="test execution tutorials"
script:
- "scripts/build_assembly_no_test.sh execution-tutorial"
- "scripts/test_execution_tutorial.sh"
52 changes: 52 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,57 @@
# Scalding #

### Version 0.15.0 ###
* Move OrderedSerialization into zero-dep scalding-serialization module #1289
* bump elephantbird to 4.8 #1292
* Fix OrderedSerialization for some forked graphs #1293
* Add serialization modules to aggregate list #1298

### Version 0.14.0 ###
* add .unit to Execution object #1189
* Override hashCode for Args #1190
* Put a value in a exception message #1191
* Add an exclusiveUpper method to DateRange #1194
* Covert LzoTextDelimited to Cascading scheme. #1179
* Remove Travis IRC notifications #1200
* add LookupJoin and LookupJoinTest changes from summingbird #1199
* Add a new ExecutionApp tutorial #1196
* Move main simple example to be the typed API, and put the .'s at the sta... #1193
* Add Execution.withArgs #1205
* Config/Cascading updater #1197
* Remove algebird serializers #1206
* remove warnings in CumulativeSum #1215
* Implicit execution context / easier switching between modes #1113
* add row l1 normalize #1214
* provide Args as an implicit val #1219
* call sourceConfInit when reading from taps in local mode #1228
* Add distinctCount and distinctValues helper methods to KeyedList. #1232
* import hygiene: remove unused imports and remove JavaConversions use #1239
* Swap hash and filename for filename-extension-sensitive code #1243
* Remove more unused imports #1240
* Provide useHdfsLocalMode for an easy switch to mapreduce local mode #1244
* upgrade scalacheck and scalatest #1246
* Optimize string and (hopefully) number comparisons a bit #1241
* Note the active FlowProcess for Joiners #1235
* Make sure Executions are executed at most once #1253
* Fix Config.getUniqueIDs #1254
* Add MustHasReducers trait. #1252
* Make sure the EvalCache thread isDaemon #1255
* Use non-regex split function #1251
* make InputSizeReducerEstimator work for any CompositeTap #1256
* TimePathedSource helper methods #1257
* Fix for reducer estimation not working correctly if withReducers is set to 1 reducer #1263
* Add make(dest) to TypedPipe #1217
* Fix SimpleDateFormat caching by default #1265
* upgrade sbt and sbt launcher script #1270
* Add TypedPipeDiff for comparing typed pipes #1266
* Change separator from \1 to \u0001 #1271
* Disable reducer estimation for map-only steps #1276
* Local sources support multiple paths #1275
* fix the spelling of the cumulativeSumTest file #1281
* Hydrate both sides of sampledCounts in skewJoinWithSmaller #1278
* Bijection 0.8.0, algebird 0.10.0, chill 0.6.0, scala 2.10.5 #1287
* Remove some deprecated items #1288

### Version 0.13.1 ###
* Back out 4 changes to be binary compatible: https://github.com/twitter/scalding/pull/1187
* Use java.util.Random instead of scala.util.Random: https://github.com/twitter/scalding/pull/1186
Expand Down
11 changes: 8 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Scalding is a Scala library that makes it easy to specify Hadoop MapReduce jobs.

![Scalding Logo](https://raw.github.com/twitter/scalding/develop/logo/scalding.png)

Current version: `0.13.1`
Current version: `0.15.0`

## Word Count

Expand Down Expand Up @@ -37,16 +37,17 @@ You can find more example code under [examples/](https://github.com/twitter/scal
## Documentation and Getting Started

* [**Getting Started**](https://github.com/twitter/scalding/wiki/Getting-Started) page on the [Scalding Wiki](https://github.com/twitter/scalding/wiki)
* [Scalding Scaladocs](http://twitter.github.com/scalding) provide details beyond the API References. Prefer using this as it's always up to date.
* [**REPL in Wonderland**](https://gist.github.com/johnynek/a47699caa62f4f38a3e2) a hands-on tour of the
scalding REPL requiring only git and java installed.
* [**Runnable tutorials**](https://github.com/twitter/scalding/tree/master/tutorial) in the source.
* The API Reference, including many example Scalding snippets:
* [Type-safe API Reference](https://github.com/twitter/scalding/wiki/Type-safe-api-reference)
* [Fields-based API Reference](https://github.com/twitter/scalding/wiki/Fields-based-API-Reference)
* [Scalding Scaladocs](http://twitter.github.com/scalding) provide details beyond the API References
* The Matrix Library provides a way of working with key-attribute-value scalding pipes:
* The [Introduction to Matrix Library](https://github.com/twitter/scalding/wiki/Introduction-to-Matrix-Library) contains an overview and a "getting started" example
* The [Matrix API Reference](https://github.com/twitter/scalding/wiki/Matrix-API-Reference) contains the Matrix Library API reference with examples
* [**Introduction to Scalding Execution**](https://github.com/twitter/scalding/wiki/Calling-Scalding-from-inside-your-application) contains general rules and examples of calling Scalding from inside another application.

Please feel free to use the beautiful [Scalding logo](https://drive.google.com/folderview?id=0B3i3pDi3yVgNbm9pMUdDcHFKVEk&usp=sharing) artwork anywhere.

Expand Down Expand Up @@ -124,6 +125,10 @@ Thanks for assistance and contributions:

* Sam Ritchie <http://twitter.com/sritchie>
* Aaron Siegel: <http://twitter.com/asiegel>
* Ian O'Connell <http://twitter.com/0x138>
* Alex Levenson <http://twitter.com/THISWILLWORK>
* Jonathan Coveney <http://twitter.com/jco>
* Kevin Lin <http://twitter.com/reconditesea>
* Brad Greenlee: <http://twitter.com/bgreenlee>
* Edwin Chen <http://twitter.com/edchedch>
* Arkajit Dey: <http://twitter.com/arkajit>
Expand All @@ -133,9 +138,9 @@ Thanks for assistance and contributions:
* Ning Liang <http://twitter.com/ningliang>
* Dmitriy Ryaboy <http://twitter.com/squarecog>
* Dong Wang <http://twitter.com/dongwang218>
* Kevin Lin <http://twitter.com/reconditesea>
* Josh Attenberg <http://twitter.com/jattenberg>
* Juliet Hougland <https://twitter.com/j_houg>
* Eddie Xie <https://twitter.com/eddiex>
A full list of [contributors](https://github.com/twitter/scalding/graphs/contributors) can be found on GitHub.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.RecordReader;
import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
3 changes: 0 additions & 3 deletions maple/src/main/java/com/twitter/maple/hbase/HBaseTap.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import cascading.flow.FlowProcess;
import cascading.tap.SinkMode;
import cascading.tap.Tap;
import cascading.tap.hadoop.io.HadoopTupleEntrySchemeCollector;
import cascading.tap.hadoop.io.HadoopTupleEntrySchemeIterator;
import cascading.tuple.TupleEntryCollector;
import cascading.tuple.TupleEntryIterator;
Expand All @@ -33,10 +32,8 @@
import org.apache.hadoop.mapred.RecordReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.reflect.generics.reflectiveObjects.NotImplementedException;

import java.io.IOException;
import java.util.Map.Entry;
import java.util.UUID;

/**
Expand Down
Loading

0 comments on commit afbde5a

Please sign in to comment.