Skip to content
This repository has been archived by the owner on Jan 27, 2020. It is now read-only.

Commit

Permalink
Factored Oozie, Shell, and Pig drivers / transformations into separat…
Browse files Browse the repository at this point in the history 
…e submodules to allow people not using

these transformations to leave their dependencies out of their classpath
  • Loading branch information
Utz Westermann committed Aug 18, 2016
1 parent 0974233 commit 0a9cdc8
Show file tree
Hide file tree
Showing 55 changed files with 1,047 additions and 589 deletions.
3 changes: 3 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<module>schedoscope-conf</module>
<module>schedoscope-export</module>
<module>schedoscope-core</module>
<module>schedoscope-transformation-oozie</module>
<module>schedoscope-transformation-pig</module>
<module>schedoscope-transformation-shell</module>
<module>schedoscope-tutorial</module>
<module>schedoscope-metascope</module>
</modules>
Expand Down
178 changes: 0 additions & 178 deletions schedoscope-conf/src/main/resources/reference.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,65 +519,6 @@ schedoscope {

},

#
# Settings for Pig transformations
#

pig: {

#
# Class implementing the Pig driver
#

driverClassName = "org.schedoscope.scheduler.driver.PigDriver"

#
# Location where to put Pig library jar in HDFS
#

location = "/tmp/soda/pig/"

#
# Ignored
#

libDirectory = ""

#
# Ignored.
#

url = ""

#
# Do not change. Pig jars should not be unpacked in HDFS.
#

unpack = false

#
# Number of parallel Driver actors to use for executing Pig
# transformations
#

concurrency = 10

#
# Timeout for Pig transformations.
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# Settings for MapReduce transformations
#
Expand Down Expand Up @@ -638,67 +579,6 @@ schedoscope {

},

#
# Oozie Driver settings
#

oozie: {

#
# Class implementing the Oozie driver
#

driverClassName = "org.schedoscope.scheduler.driver.OozieDriver"

#
# Where to put Oozie bundles in HDFS
#

location = "/tmp/schedoscope/oozie/"

#
# Comma-separated list of directories where additional Oozie workflow
# bundle jars can be found that are to be put onto HDFS when launching
# Schedoscope.
#

libDirectory = ""

#
# URL of Oozie Server
#

url = "http://localhost:11000/oozie"

#
# Number of parallel Driver actors to use for executing Oozie
# transformations
#

concurrency = 10

#
# Oozie bundle jars need to be unpacked in HDFS.
#

unpack = true

#
# Timeout for Oozie transformations
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# File system driver settings
#
Expand Down Expand Up @@ -815,65 +695,7 @@ schedoscope {

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]

},

#
# Shell driver settings
#

shell: {

#
# Class implementing the Shell driver
#

driverClassName = "org.schedoscope.scheduler.driver.ShellDriver"

#
# Number of parallel Shell Driver actors to use
#

concurrency = 1

#
# Ignored
#

location = "/"

#
# Ignored
#

libDirectory = ""

#
# Ignored
#

url = ""

#
# Ignored
#

unpack = false

#
# Timeout for Shell transformations
#

timeout = 1 day

#
# The handlers being notified after each driver run has
# finished (succeeded or failed). These must implement the
# trait org.schedoscope.scheduler.driver.DriverRunCompletionHandler
#

driverRunCompletionHandlers = ["org.schedoscope.scheduler.driver.DoNothingCompletionHandler"]
}

}
}

Expand Down
100 changes: 23 additions & 77 deletions schedoscope-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@
<artifactId>schedoscope-conf</artifactId>
<version>${schedoscope.version}</version>
</dependency>
<dependency>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
<version>11.0</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
Expand Down Expand Up @@ -120,6 +125,12 @@
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-contrib_${scala.version}</artifactId>
<version>${akka.version}</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
Expand Down Expand Up @@ -159,6 +170,10 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand All @@ -174,20 +189,9 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
Expand All @@ -201,71 +205,9 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-pig-bundle</artifactId>
<version>${pig.parquet.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
<artifactId>hive-hcatalog-pig-adapter</artifactId>
<version>${hive.hcatalog.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
<artifactId>oozie-client</artifactId>
<version>${oozie.version}</version>
<exclusions>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-simple</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>minioozie</groupId>
<artifactId>minioozie</artifactId>
<version>${minioozie.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
Expand Down Expand Up @@ -298,6 +240,10 @@
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down
Loading

0 comments on commit 0a9cdc8

Please sign in to comment.