Skip to content
This repository has been archived by the owner on Jan 27, 2020. It is now read-only.

Commit

Permalink
Merge pull request #105 from utzwestermann/master
Browse files Browse the repository at this point in the history 
Factored Oozie, Pig, and Shell Transformations / Drivers into Separate Modules
  • Loading branch information
@utzwestermann
utzwestermann authored Aug 18, 2016
2 parents 0974233 + f629fc9 commit 03354ca
Show file tree
Hide file tree
Showing 55 changed files with 1,047 additions and 589 deletions.
3 changes: 3 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<module>schedoscope-conf</module>
<module>schedoscope-export</module>
<module>schedoscope-core</module>
<module>schedoscope-transformation-oozie</module>
<module>schedoscope-transformation-pig</module>
<module>schedoscope-transformation-shell</module>
<module>schedoscope-tutorial</module>
<module>schedoscope-metascope</module>
</modules>
Expand Down
178 changes: 0 additions & 178 deletions schedoscope-conf/src/main/resources/reference.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,65 +519,6 @@ schedoscope {

},

#
# Settings for Pig transformations
#

pig: {

#
# Class implementing the Pig driver
#

driverClassName = "org.schedoscope.scheduler.driver.PigDriver"

#
# Location where to put Pig library jar in HDFS
#

location = "/tmp/soda/pig/"

#
# Ignored
#

libDirectory = ""

#
# Ignored.
#

url = ""

#
# Do not change. Pig jars should not be unpacked in HDFS.
#

unpack = false

#
# Number of parallel Driver actors to use for executing Pig
# transformations
#

concurrency = 10

#
# Timeout for Pig transformations.
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# Settings for MapReduce transformations
#
Expand Down Expand Up @@ -638,67 +579,6 @@ schedoscope {

},

#
# Oozie Driver settings
#

oozie: {

#
# Class implementing the Oozie driver
#

driverClassName = "org.schedoscope.scheduler.driver.OozieDriver"

#
# Where to put Oozie bundles in HDFS
#

location = "/tmp/schedoscope/oozie/"

#
# Comma-separated list of directories where additional Oozie workflow
# bundle jars can be found that are to be put onto HDFS when launching
# Schedoscope.
#

libDirectory = ""

#
# URL of Oozie Server
#

url = "http://localhost:11000/oozie"

#
# Number of parallel Driver actors to use for executing Oozie
# transformations
#

concurrency = 10

#
# Oozie bundle jars need to be unpacked in HDFS.
#

unpack = true

#
# Timeout for Oozie transformations
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# File system driver settings
#
Expand Down Expand Up @@ -815,65 +695,7 @@ schedoscope {

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# Shell driver settings
#

shell: {

#
# Class implementing the Shell driver
#

driverClassName = "org.schedoscope.scheduler.driver.ShellDriver"

#
# Number of parallel Shell Driver actors to use
#

concurrency = 1

#
# Ignored
#

location = "/"

#
# Ignored
#

libDirectory = ""

#
# Ignored
#

url = ""

#
# Ignored
#

unpack = false

#
# Timeout for Shell transformations
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]
}

}
}

Expand Down
100 changes: 23 additions & 77 deletions schedoscope-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@
<artifactId>schedoscope-conf</artifactId>
<version>${schedoscope.version}</version>
</dependency>
<dependency>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
<version>11.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
Expand Down Expand Up @@ -120,6 +125,12 @@
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-contrib_${scala.version}</artifactId>
<version>${akka.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
Expand Down Expand Up @@ -159,6 +170,10 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -174,20 +189,9 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
Expand All @@ -201,71 +205,9 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-pig-bundle</artifactId>
<version>${pig.parquet.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-hcatalog-pig-adapter</artifactId>
<version>${hive.hcatalog.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-client</artifactId>
<version>${oozie.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-simple</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>minioozie</groupId>
<artifactId>minioozie</artifactId>
<version>${minioozie.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
Expand Down Expand Up @@ -298,6 +240,10 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down
Loading

0 comments on commit 03354ca

Please sign in to comment.