Skip to content

Commit

Permalink
Debugging Roll-Up reports...
Browse files Browse the repository at this point in the history
  • Loading branch information
simonambridge committed Nov 20, 2017
1 parent 74b4ed1 commit 02b7eb3
Show file tree
Hide file tree
Showing 73 changed files with 502 additions and 498 deletions.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#default#rollupreports-build;0.1-SNAPSHOT resolved revisions
#Mon Nov 13 13:13:16 GMT 2017
#Mon Nov 20 18:41:28 GMT 2017
+organisation\:\#@\#\:+org.scala-lang\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+scala-compiler\:\#@\#\:+revision\:\#@\#\:+2.10.4\:\#@\#\:=2.10.4 release 2.10.4 null
+organisation\:\#@\#\:+org.scala-lang\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+scala-library\:\#@\#\:+revision\:\#@\#\:+2.10.4\:\#@\#\:=2.10.4 ? 2.10.4 null
+organisation\:\#@\#\:+org.scala-sbt\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+sbt\:\#@\#\:+revision\:\#@\#\:+0.13.8\:\#@\#\:=0.13.8 release 0.13.8 null
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
module="rollupreports-build"
revision="0.1-SNAPSHOT"
status="integration"
publication="20171113131315"
publication="20171120184127"
e:sbtVersion="0.13" e:scalaVersion="2.10"
>
<description>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="compile-internal"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
<module organisation="org.scala-sbt" name="sbt">
<revision name="0.13.8" status="release" pubdate="20150320121147" resolver="sbt-chain" artresolver="sbt-chain" downloaded="false" searched="false" default="false" conf="compile, default(compile)" position="1">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="compile"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="docs"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="optional"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="plugin"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="pom"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="provided"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
<module organisation="org.scala-sbt" name="sbt">
<revision name="0.13.8" status="release" pubdate="20150320121147" resolver="sbt-chain" artresolver="sbt-chain" downloaded="false" searched="false" default="false" conf="compile, default(compile)" position="1">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="runtime-internal"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="runtime"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="scala-tool"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
<module organisation="org.scala-lang" name="scala-compiler">
<revision name="2.10.4" status="release" pubdate="20140318214738" resolver="sbt-chain" artresolver="sbt-chain" homepage="http://www.scala-lang.org/" downloaded="false" searched="false" default="false" conf="default, compile, runtime, optional, optional(default), master" position="0">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="sources"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="test-internal"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
<module organisation="org.scala-sbt" name="sbt">
<revision name="0.13.8" status="release" pubdate="20150320121147" resolver="sbt-chain" artresolver="sbt-chain" downloaded="false" searched="false" default="false" conf="compile, default(compile)" position="1">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
extra-sbtVersion="0.13"
conf="test"
confs="compile, runtime, test, provided, optional, compile-internal, runtime-internal, test-internal, plugin, sources, docs, pom, scala-tool"
date="20171113131315"/>
date="20171120184127"/>
<dependencies>
</dependencies>
</ivy-report>
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
[debug] == resolving dependencies for default#rollupreports-build;0.1-SNAPSHOT [provided]
[debug] == resolving dependencies default#rollupreports-build;0.1-SNAPSHOT->org.scala-lang#scala-library;2.10.4 [provided->default(compile)]
[info] Resolving org.scala-lang#scala-library;2.10.4 ...
[debug] sbt-chain: Checking cache for: sbt.ivyint.MergedDescriptors@3b0891da
[debug] sbt-chain: Checking cache for: sbt.ivyint.MergedDescriptors@38692ea4
[debug] sbt-chain: module revision found in cache: org.scala-lang#scala-library;2.10.4
[debug]  found org.scala-lang#scala-library;2.10.4 in sbt-chain
[debug]  [2.10.4] org.scala-lang#scala-library;2.10.4
Expand Down Expand Up @@ -1766,7 +1766,7 @@
[debug]  [NOT REQUIRED] org.scala-sbt#apply-macro;0.13.8!apply-macro.jar
[debug]  [NOT REQUIRED] org.scala-lang#jline;2.10.4!jline.jar
[debug]  [NOT REQUIRED] org.fusesource.jansi#jansi;1.4!jansi.jar
[debug] :: resolution report :: resolve 1042ms :: artifacts dl 24ms
[debug] :: resolution report :: resolve 949ms :: artifacts dl 32ms
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT compile produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-compile.xml
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT runtime produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-runtime.xml
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT test produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-test.xml
Expand All @@ -1780,5 +1780,5 @@
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT docs produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-docs.xml
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT pom produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-pom.xml
[debug]  report for default#rollupreports-build;0.1-SNAPSHOT scala-tool produced in /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/project/target/resolution-cache/reports/default-rollupreports-build-scala-tool.xml
[debug]  resolve done (1042ms resolve - 24ms download)
[debug]  resolve done (949ms resolve - 32ms download)
[info] Done updating.
Binary file not shown.
Binary file not shown.
27 changes: 14 additions & 13 deletions RollUpReports/src/main/scala/RollUpReportsByMerchant.scala
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ object RollUpReportsByMerchant {
def main (args: Array[String]){
println("Beginning RollUp Reporting By Merchant...")

val conf = new SparkConf().setAppName("RollUpReportsByMerchant")

// val conf = new SparkConf().setAppName("RollUpReportsByMerchant")
val conf = new SparkConf().setAppName("NetworkWordCount").set("spark.ui.port", "40400" ).set("spark.driver.allowMultipleContexts", "true")
val sc = SparkContext.getOrCreate(conf)
val sqlContext = new HiveContext(sc)

Expand All @@ -29,18 +29,19 @@ object RollUpReportsByMerchant {
pushdown "true"
)""")

val rollup1= sqlContext.sql("select txn_time, " +
"cc_no, amount, " +
val rollup1= sqlContext.sql("select merchant, " +
"int(translate(string(date(txn_time)),'-','')) as day, " +
"cc_no, " +
"txn_id, " +
"txn_time, " +
"amount, " +
"cc_provider, " +
"items, " +
"location, " +
"merchant, " +
"notes, " +
"status, " +
"txn_id, " +
"user_id, " +
"tags, " +
"int(translate(string(date(txn_time)),'-','')) as day " +
"tags, " +
"user_id " +
"from temp_transactions")

rollup1.write.format("org.apache.spark.sql.cassandra")
Expand All @@ -61,14 +62,14 @@ object RollUpReportsByMerchant {
)""")

sqlContext.udf.register("now", () => System.currentTimeMillis)
val rollup2= sqlContext.sql("select date(timestamp(now())) as txn_time, " +
"string(now()) as txn_id, " +
"merchant, day, count(*) as total_count, " +
val rollup2= sqlContext.sql("select merchant, " +
"day, " +
"count(*) as total_count, " +
"sum(amount) as total_amount, " +
"min(amount) as min_amount, " +
"max(amount) as max_amount " +
"from temp_dailytxns_bymerchant " +
"group by merchant, day")
"group by merchant, day order by merchant")

rollup2.write.format("org.apache.spark.sql.cassandra")
.mode(SaveMode.Append)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#default#rollupreports_2.11;1.0 resolved revisions
#Mon Nov 13 13:17:45 GMT 2017
#Mon Nov 20 18:41:41 GMT 2017
+organisation\:\#@\#\:+org.apache.spark\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+spark-core_2.11\:\#@\#\:+revision\:\#@\#\:+2.0.2\:\#@\#\:=2.0.2 release 2.0.2 null
+organisation\:\#@\#\:+org.scala-lang\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+scala-library\:\#@\#\:+revision\:\#@\#\:+2.11.8\:\#@\#\:=2.11.8 ? 2.11.8 null
+organisation\:\#@\#\:+org.apache.spark\:\#@\#\:+branch\:\#@\#\:+@\#\:NULL\:\#@\:\#@\#\:+module\:\#@\#\:+spark-hive_2.11\:\#@\#\:+revision\:\#@\#\:+2.0.2\:\#@\#\:=2.0.2 release 2.0.2 null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
module="rollupreports_2.11"
revision="1.0"
status="integration"
publication="20171113131736"
publication="20171120184130"
>
<description>
RollUpReports
Expand Down
Loading

0 comments on commit 02b7eb3

Please sign in to comment.