Reports and stress test completed

simonambridge · Nov 21, 2017 · 86b2aaf · 86b2aaf
1 parent 9c005c3
commit 86b2aaf
Show file tree

Hide file tree

Showing 84 changed files with 1,636 additions and 33 deletions.
diff --git a/RollUpReports/derby.log b/RollUpReports/derby.log
@@ -1,7 +1,7 @@
 ----------------------------------------------------------------
-Mon Nov 20 23:10:10 GMT 2017:
-Booting Derby version The Apache Software Foundation - Apache Derby - 10.10.2.0 - (1582446): instance a816c00e-015f-dbb2-3637-000033868cc0 
-on database directory /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/metastore_db with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@11adca8f 
+Tue Nov 21 12:59:57 GMT 2017:
+Booting Derby version The Apache Software Foundation - Apache Derby - 10.10.2.0 - (1582446): instance a816c00e-015f-dea9-e442-0000328fcfd8 
+on database directory /Users/simonambridge/Documents/My Projects/GitHub/RTFAP2/RollUpReports/metastore_db with class loader org.apache.spark.sql.hive.client.IsolatedClientLoader$$anon$1@4a204364 
 Loaded from file:/Users/simonambridge/zeppelin-0.7.1/interpreter/spark/dep/zeppelin-spark-dependencies_2.11-0.7.1.jar
 java.vendor=Oracle Corporation
 java.runtime.version=1.8.0_152-b16

diff --git a/RollUpReports/metastore_db/db.lck b/RollUpReports/metastore_db/db.lck
diff --git a/RollUpReports/metastore_db/log/log.ctrl b/RollUpReports/metastore_db/log/log.ctrl
diff --git a/RollUpReports/metastore_db/log/log1.dat b/RollUpReports/metastore_db/log/log1.dat
diff --git a/RollUpReports/metastore_db/log/logmirror.ctrl b/RollUpReports/metastore_db/log/logmirror.ctrl
diff --git a/RollUpReports/metastore_db/seg0/c101.dat b/RollUpReports/metastore_db/seg0/c101.dat
diff --git a/RollUpReports/metastore_db/seg0/c111.dat b/RollUpReports/metastore_db/seg0/c111.dat
diff --git a/RollUpReports/metastore_db/seg0/c121.dat b/RollUpReports/metastore_db/seg0/c121.dat
diff --git a/RollUpReports/metastore_db/seg0/c130.dat b/RollUpReports/metastore_db/seg0/c130.dat
diff --git a/RollUpReports/metastore_db/seg0/c141.dat b/RollUpReports/metastore_db/seg0/c141.dat
diff --git a/RollUpReports/metastore_db/seg0/c150.dat b/RollUpReports/metastore_db/seg0/c150.dat
diff --git a/RollUpReports/metastore_db/seg0/c161.dat b/RollUpReports/metastore_db/seg0/c161.dat
diff --git a/RollUpReports/metastore_db/seg0/c171.dat b/RollUpReports/metastore_db/seg0/c171.dat
diff --git a/RollUpReports/metastore_db/seg0/c180.dat b/RollUpReports/metastore_db/seg0/c180.dat
diff --git a/RollUpReports/metastore_db/seg0/c191.dat b/RollUpReports/metastore_db/seg0/c191.dat
diff --git a/RollUpReports/metastore_db/seg0/c1a1.dat b/RollUpReports/metastore_db/seg0/c1a1.dat
diff --git a/RollUpReports/metastore_db/seg0/c1b1.dat b/RollUpReports/metastore_db/seg0/c1b1.dat
diff --git a/RollUpReports/metastore_db/seg0/c1e0.dat b/RollUpReports/metastore_db/seg0/c1e0.dat
diff --git a/RollUpReports/metastore_db/seg0/c1f1.dat b/RollUpReports/metastore_db/seg0/c1f1.dat
diff --git a/RollUpReports/metastore_db/seg0/c20.dat b/RollUpReports/metastore_db/seg0/c20.dat
diff --git a/RollUpReports/metastore_db/seg0/c200.dat b/RollUpReports/metastore_db/seg0/c200.dat
diff --git a/RollUpReports/metastore_db/seg0/c211.dat b/RollUpReports/metastore_db/seg0/c211.dat
diff --git a/RollUpReports/metastore_db/seg0/c221.dat b/RollUpReports/metastore_db/seg0/c221.dat
diff --git a/RollUpReports/metastore_db/seg0/c230.dat b/RollUpReports/metastore_db/seg0/c230.dat
diff --git a/RollUpReports/metastore_db/seg0/c31.dat b/RollUpReports/metastore_db/seg0/c31.dat
diff --git a/RollUpReports/metastore_db/seg0/c41.dat b/RollUpReports/metastore_db/seg0/c41.dat
diff --git a/RollUpReports/metastore_db/seg0/c490.dat b/RollUpReports/metastore_db/seg0/c490.dat
diff --git a/RollUpReports/metastore_db/seg0/c4a1.dat b/RollUpReports/metastore_db/seg0/c4a1.dat
diff --git a/RollUpReports/metastore_db/seg0/c4b0.dat b/RollUpReports/metastore_db/seg0/c4b0.dat
diff --git a/RollUpReports/metastore_db/seg0/c4d1.dat b/RollUpReports/metastore_db/seg0/c4d1.dat
diff --git a/RollUpReports/metastore_db/seg0/c4f0.dat b/RollUpReports/metastore_db/seg0/c4f0.dat
diff --git a/RollUpReports/metastore_db/seg0/c51.dat b/RollUpReports/metastore_db/seg0/c51.dat
diff --git a/RollUpReports/metastore_db/seg0/c510.dat b/RollUpReports/metastore_db/seg0/c510.dat
diff --git a/RollUpReports/metastore_db/seg0/c530.dat b/RollUpReports/metastore_db/seg0/c530.dat
diff --git a/RollUpReports/metastore_db/seg0/c550.dat b/RollUpReports/metastore_db/seg0/c550.dat
diff --git a/RollUpReports/metastore_db/seg0/c570.dat b/RollUpReports/metastore_db/seg0/c570.dat
diff --git a/RollUpReports/metastore_db/seg0/c590.dat b/RollUpReports/metastore_db/seg0/c590.dat
diff --git a/RollUpReports/metastore_db/seg0/c5b0.dat b/RollUpReports/metastore_db/seg0/c5b0.dat
diff --git a/RollUpReports/metastore_db/seg0/c5d0.dat b/RollUpReports/metastore_db/seg0/c5d0.dat
diff --git a/RollUpReports/metastore_db/seg0/c5f0.dat b/RollUpReports/metastore_db/seg0/c5f0.dat
diff --git a/RollUpReports/metastore_db/seg0/c60.dat b/RollUpReports/metastore_db/seg0/c60.dat
diff --git a/RollUpReports/metastore_db/seg0/c610.dat b/RollUpReports/metastore_db/seg0/c610.dat
diff --git a/RollUpReports/metastore_db/seg0/c630.dat b/RollUpReports/metastore_db/seg0/c630.dat
diff --git a/RollUpReports/metastore_db/seg0/c650.dat b/RollUpReports/metastore_db/seg0/c650.dat
diff --git a/RollUpReports/metastore_db/seg0/c670.dat b/RollUpReports/metastore_db/seg0/c670.dat
diff --git a/RollUpReports/metastore_db/seg0/c690.dat b/RollUpReports/metastore_db/seg0/c690.dat
diff --git a/RollUpReports/metastore_db/seg0/c6b0.dat b/RollUpReports/metastore_db/seg0/c6b0.dat
diff --git a/RollUpReports/metastore_db/seg0/c6d0.dat b/RollUpReports/metastore_db/seg0/c6d0.dat
diff --git a/RollUpReports/metastore_db/seg0/c6f0.dat b/RollUpReports/metastore_db/seg0/c6f0.dat
diff --git a/RollUpReports/metastore_db/seg0/c71.dat b/RollUpReports/metastore_db/seg0/c71.dat
diff --git a/RollUpReports/metastore_db/seg0/c81.dat b/RollUpReports/metastore_db/seg0/c81.dat
diff --git a/RollUpReports/metastore_db/seg0/c840.dat b/RollUpReports/metastore_db/seg0/c840.dat
diff --git a/RollUpReports/metastore_db/seg0/c860.dat b/RollUpReports/metastore_db/seg0/c860.dat
diff --git a/RollUpReports/metastore_db/seg0/c880.dat b/RollUpReports/metastore_db/seg0/c880.dat
diff --git a/RollUpReports/metastore_db/seg0/c8a0.dat b/RollUpReports/metastore_db/seg0/c8a0.dat
diff --git a/RollUpReports/metastore_db/seg0/c90.dat b/RollUpReports/metastore_db/seg0/c90.dat
diff --git a/RollUpReports/metastore_db/seg0/c920.dat b/RollUpReports/metastore_db/seg0/c920.dat
diff --git a/RollUpReports/metastore_db/seg0/c931.dat b/RollUpReports/metastore_db/seg0/c931.dat
diff --git a/RollUpReports/metastore_db/seg0/c940.dat b/RollUpReports/metastore_db/seg0/c940.dat
diff --git a/RollUpReports/metastore_db/seg0/c951.dat b/RollUpReports/metastore_db/seg0/c951.dat
diff --git a/RollUpReports/metastore_db/seg0/c960.dat b/RollUpReports/metastore_db/seg0/c960.dat
diff --git a/RollUpReports/metastore_db/seg0/c971.dat b/RollUpReports/metastore_db/seg0/c971.dat
diff --git a/RollUpReports/metastore_db/seg0/c981.dat b/RollUpReports/metastore_db/seg0/c981.dat
diff --git a/RollUpReports/metastore_db/seg0/c990.dat b/RollUpReports/metastore_db/seg0/c990.dat
diff --git a/RollUpReports/metastore_db/seg0/c9a1.dat b/RollUpReports/metastore_db/seg0/c9a1.dat
diff --git a/RollUpReports/metastore_db/seg0/c9b1.dat b/RollUpReports/metastore_db/seg0/c9b1.dat
diff --git a/RollUpReports/metastore_db/seg0/c9c0.dat b/RollUpReports/metastore_db/seg0/c9c0.dat
diff --git a/RollUpReports/metastore_db/seg0/c9e0.dat b/RollUpReports/metastore_db/seg0/c9e0.dat
diff --git a/RollUpReports/metastore_db/seg0/ca1.dat b/RollUpReports/metastore_db/seg0/ca1.dat
diff --git a/RollUpReports/metastore_db/seg0/cb1.dat b/RollUpReports/metastore_db/seg0/cb1.dat
diff --git a/RollUpReports/metastore_db/seg0/cf0.dat b/RollUpReports/metastore_db/seg0/cf0.dat
diff --git a/RollUpReports/src/main/scala/RollUpReportsByCC.scala b/RollUpReports/src/main/scala/RollUpReportsByCC.scala
@@ -34,7 +34,7 @@ object RollUpReportsByCC {
     rollupDF.registerTempTable("my_transactions")
 
     // 1. hourlyaggregates_bycc
-    println(" - Populating hourlyaggregates_bycc")
+    println(" - 1. Populating hourlyaggregates_bycc")
 
     val rollup1= sqlContext.sql("select cc_no, " +
       "int(concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day), if(length(hour)=1, concat('0',hour), hour))) as hour, " +
@@ -52,12 +52,13 @@ object RollUpReportsByCC {
       .save()
 
     // 2. dailyaggregates_bycc
-    println(" - Populating dailyaggregates_bycc")
+    println(" - 2. Populating dailyaggregates_bycc")
     val rollup2= sqlContext.sql("select cc_no, " +
       "int(concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day))) as day, " +
       "sum(amount) as total_amount, " +
       "min(amount) as min_amount, " +
-      "max(amount) as max_amount " +
+      "max(amount) as max_amount, " +
+      "count(*) as total_count " +
       "from my_transactions " +
       "group by cc_no, " +
       "concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day))")
@@ -68,12 +69,13 @@ object RollUpReportsByCC {
       .save()
 
     // 3. monthlyaggregates_bycc
-    println(" - Populating monthlyaggregates_bycc")
+    println(" - 3. Populating monthlyaggregates_bycc")
     val rollup3= sqlContext.sql("select cc_no, " +
       "int(concat(year, if(length(month)=1, concat('0',month), month))) as month, " +
       "sum(amount) as total_amount, " +
       "min(amount) as min_amount, " +
-      "max(amount) as max_amount " +
+      "max(amount) as max_amount, " +
+      "count(*) as total_count " +
       "from my_transactions " +
       "group by cc_no, concat(year, if(length(month)=1, concat('0',month), month))")
 
@@ -83,7 +85,7 @@ object RollUpReportsByCC {
       .save()
 
     // 4. yearlyaggregates_bycc
-    println(" - Populating yearlyaggregates_bycc")
+    println(" - 4. Populating yearlyaggregates_bycc")
     val rollup4= sqlContext.sql("select cc_no, " +
       "int(year) as year, " +
       "sum(amount) as total_amount, " +

diff --git a/RollUpReports/target/scala-2.11/classes/RollUpReportsByCC$.class b/RollUpReports/target/scala-2.11/classes/RollUpReportsByCC$.class
diff --git a/RollUpReports/target/scala-2.11/classes/RollUpReportsByMerchant$$anonfun$main$1.class b/RollUpReports/target/scala-2.11/classes/RollUpReportsByMerchant$$anonfun$main$1.class
diff --git a/RollUpReports/target/scala-2.11/classes/RollUpReportsByMerchant$.class b/RollUpReports/target/scala-2.11/classes/RollUpReportsByMerchant$.class
diff --git a/RollUpReports/target/scala-2.11/rollupreports_2.11-1.0.jar b/RollUpReports/target/scala-2.11/rollupreports_2.11-1.0.jar
diff --git a/...s/target/streams/$global/dependencyPositions/$global/streams/update_cache_2.11/output_dsp b/...s/target/streams/$global/dependencyPositions/$global/streams/update_cache_2.11/output_dsp
diff --git a/RollUpReports/target/streams/compile/compileIncremental/$global/streams/export b/RollUpReports/target/streams/compile/compileIncremental/$global/streams/export
diff --git a/RollUpReports/target/streams/compile/compileIncremental/$global/streams/out b/RollUpReports/target/streams/compile/compileIncremental/$global/streams/out
diff --git a/RollUpReports/target/streams/compile/incCompileSetup/$global/streams/inc_compile_2.11 b/RollUpReports/target/streams/compile/incCompileSetup/$global/streams/inc_compile_2.11
diff --git a/RollUpReports/target/streams/compile/packageBin/$global/streams/inputs b/RollUpReports/target/streams/compile/packageBin/$global/streams/inputs
diff --git a/creates_and_inserts.cql b/creates_and_inserts.cql
@@ -119,6 +119,28 @@ CREATE TABLE if not exists rtfap.txn_count_min (
 	PRIMARY KEY (( year, month, day, hour, minute ))
 );
 
+// Cassandra stress table
+
+CREATE TABLE if not exists rtfap.txn_by_cc (
+cc_no text,
+txn_year int,
+txn_month int,
+txn_day int,
+txn_time timestamp,
+amount double,
+cc_provider text,
+location text,
+merchant text,
+notes text,
+status text,
+tags set<text>,
+txn_id text,
+user_id text,
+	PRIMARY KEY (( cc_no, txn_year, txn_month, txn_day ), txn_time)
+) WITH CLUSTERING ORDER BY ( txn_time DESC )
+AND default_time_to_live = 86400;
+
+
 // Sample inserts
 
 insert into rtfap.transactions (year, month, day, hour, min, txn_time, cc_no, amount, cc_provider, items, location, merchant, notes, status, txn_id, user_id, tags) VALUES ( 2016, 03, 09, 11, 04, '2016-03-09 11:04:19', '1234123412341234', 200.0, 'VISA', {'tshirt':25, 'dressshirt':50, 'trousers':125}, 'San Francisco', 'Nordstrom', 'pretty good clothing', 'Approved', '098765', 'kunalak', {'Suspicious'});

diff --git a/stress_yamls/stress-read.log b/stress_yamls/stress-read.log
diff --git a/stress_yamls/stress-write.log b/stress_yamls/stress-write.log