RTFAP2 - RUP by Card.json

{"paragraphs":[{"text":"import org.apache.spark.sql.hive.HiveContext\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.{SQLContext, SaveMode}\n","user":"anonymous","dateUpdated":"2017-11-21T14:23:58+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nimport org.apache.spark.sql.hive.HiveContext\n\nimport org.apache.spark.{SparkConf, SparkContext}\n\nimport org.apache.spark.sql.{SQLContext, SaveMode}\n"}]},"apps":[],"jobName":"paragraph_1511217755663_1935191931","id":"20171118-224353_159192803","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:23:58+0000","dateFinished":"2017-11-21T14:23:59+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:1054"},{"text":"val conf = new SparkConf().setAppName(\"RollUpReportsByCard\").set(\"spark.ui.port\", \"40400\" ).set(\"spark.driver.allowMultipleContexts\", \"true\")","user":"anonymous","dateUpdated":"2017-11-21T14:27:29+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nconf: org.apache.spark.SparkConf = org.apache.spark.SparkConf@d2729d2\n"}]},"apps":[],"jobName":"paragraph_1511217755664_1945580151","id":"20171120-185113_356402705","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:27:29+0000","dateFinished":"2017-11-21T14:27:29+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1055"},{"text":"val sc = SparkContext.getOrCreate(conf)","user":"anonymous","dateUpdated":"2017-11-21T14:27:32+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nsc: org.apache.spark.SparkContext = org.apache.spark.SparkContext@25d470b7\n"}]},"apps":[],"jobName":"paragraph_1511217755664_1945580151","id":"20171118-224510_926766130","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:27:32+0000","dateFinished":"2017-11-21T14:27:32+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1056"},{"text":"val sqlContext = new HiveContext(sc)","user":"anonymous","dateUpdated":"2017-11-21T14:27:36+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nwarning: there was one deprecation warning; re-run with -deprecation for details\n\nsqlContext: org.apache.spark.sql.hive.HiveContext = org.apache.spark.sql.hive.HiveContext@77fae61\n"}]},"apps":[],"jobName":"paragraph_1511217755665_1945195402","id":"20171118-224522_1019518227","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:27:36+0000","dateFinished":"2017-11-21T14:27:36+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1057"},{"text":"println(\"Beginning RollUp Reporting By Credit Card number\")","user":"anonymous","dateUpdated":"2017-11-21T14:27:39+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"Beginning RollUp Reporting By Credit Card number\n"}]},"apps":[],"jobName":"paragraph_1511217755665_1945195402","id":"20171119-013128_10870176","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:27:39+0000","dateFinished":"2017-11-21T14:27:39+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1058"},{"text":"     sqlContext.sql(\"\"\"CREATE or REPLACE TEMPORARY VIEW temp_transactions\n      USING org.apache.spark.sql.cassandra\n      OPTIONS (\n       table \"transactions\",\n       keyspace \"rtfap\",\n       cluster \"Test Cluster\",\n       pushdown \"true\"\n     )\"\"\")\n","user":"anonymous","dateUpdated":"2017-11-21T14:27:45+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres37: org.apache.spark.sql.DataFrame = []\n"}]},"apps":[],"jobName":"paragraph_1511217755665_1945195402","id":"20171118-224534_255374190","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T14:27:45+0000","dateFinished":"2017-11-21T14:27:45+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1059"},{"text":"val txnCount = sqlContext.sql(\"SELECT count(*) from temp_transactions\")","user":"anonymous","dateUpdated":"2017-11-21T14:27:53+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\ntxnCount: org.apache.spark.sql.DataFrame = [count(1): bigint]\n"}]},"apps":[],"jobName":"paragraph_1511267692015_1803209689","id":"20171121-123452_1840893425","dateCreated":"2017-11-21T12:34:52+0000","dateStarted":"2017-11-21T14:27:53+0000","dateFinished":"2017-11-21T14:27:53+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1060"},{"text":"txnCount.show","user":"anonymous","dateUpdated":"2017-11-21T14:28:10+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------+\n|count(1)|\n+--------+\n|    4366|\n+--------+\n\n"}]},"apps":[],"jobName":"paragraph_1511267824221_1692553603","id":"20171121-123704_1845212003","dateCreated":"2017-11-21T12:37:04+0000","dateStarted":"2017-11-21T14:28:10+0000","dateFinished":"2017-11-21T14:28:10+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1061"},{"text":"// convert integer date elements to string - just a subset of fields for the CC roll-ups\nval txnDF = sqlContext.sql(\"SELECT cc_no, \" +\n    \"cast(year as String), \" +\n    \"cast(month AS String), \" +\n    \"cast(day AS String), \" +\n    \"cast(hour AS String), \" +\n    \"amount  \" +\n    \"FROM temp_transactions order by cc_no\")\n","user":"anonymous","dateUpdated":"2017-11-21T14:29:12+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\ntxnDF: org.apache.spark.sql.DataFrame = [cc_no: string, year: string ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511217957780_210638447","id":"20171120-224557_923810095","dateCreated":"2017-11-20T22:45:57+0000","dateStarted":"2017-11-21T14:29:12+0000","dateFinished":"2017-11-21T14:29:12+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1062"},{"text":"txnDF.count","user":"anonymous","dateUpdated":"2017-11-21T13:01:08+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres5: Long = 6\n"}]},"apps":[],"jobName":"paragraph_1511267793348_-1749856103","id":"20171121-123633_5189884","dateCreated":"2017-11-21T12:36:33+0000","dateStarted":"2017-11-21T13:01:08+0000","dateFinished":"2017-11-21T13:01:09+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1063"},{"text":"txnDF.show(10)","user":"anonymous","dateUpdated":"2017-11-21T14:24:22+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+----+-----+---+----+-------+\n|           cc_no|year|month|day|hour| amount|\n+----------------+----+-----+---+----+-------+\n|0007000092552438|2017|   10| 21|   2|1755.08|\n|0008000021465397|2017|   10| 21|   2|1001.77|\n|0008000083826281|2017|   10| 21|   2|2235.93|\n|0010000095396118|2017|   10| 21|   2|1010.09|\n|0017000084177332|2017|   10| 21|   2|2041.92|\n|0017000093085593|2017|   10| 21|   2| 212.19|\n|0039000016004660|2017|   10| 21|   2| 985.27|\n|0039000057860285|2017|   10| 21|   2|  369.3|\n|0040000033441800|2017|   10| 21|   2|  967.2|\n|0042000028553269|2017|   10| 21|   2|1489.74|\n+----------------+----+-----+---+----+-------+\nonly showing top 10 rows\n\n"}]},"apps":[],"jobName":"paragraph_1511221861810_1201946418","id":"20171120-235101_1464967312","dateCreated":"2017-11-20T23:51:01+0000","dateStarted":"2017-11-21T14:24:22+0000","dateFinished":"2017-11-21T14:24:23+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1064"},{"text":"    // Create a temp table based on the new DF\n    txnDF.registerTempTable(\"my_transactions\")","user":"anonymous","dateUpdated":"2017-11-21T13:01:33+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nwarning: there was one deprecation warning; re-run with -deprecation for details\n"}]},"apps":[],"jobName":"paragraph_1511218099292_659471847","id":"20171120-224819_1798930149","dateCreated":"2017-11-20T22:48:19+0000","dateStarted":"2017-11-21T13:01:33+0000","dateFinished":"2017-11-21T13:01:33+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1065"},{"text":"    println(\" - 1. Populating hourlyaggregates_bycc\")","user":"anonymous","dateUpdated":"2017-11-21T13:03:10+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":" - 1. Populating hourlyaggregates_bycc\n"}]},"apps":[],"jobName":"paragraph_1511217979839_-1260995448","id":"20171120-224619_1442331844","dateCreated":"2017-11-20T22:46:19+0000","dateStarted":"2017-11-21T13:03:10+0000","dateFinished":"2017-11-21T13:03:11+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1066"},{"text":"    val rollup1= sqlContext.sql(\"select cc_no, \" +\n      \"int(concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day), if(length(hour)=1, concat('0',hour), hour))) as hour, \" +\n      \"sum(amount) as total_amount, \" +\n      \"min(amount) as min_amount, \" +\n      \"max(amount) as max_amount, \" +\n      \"count(*) as total_count \" +\n      \"from my_transactions \" +\n      \"group by cc_no, \" +\n      \"concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day), if(length(hour)=1, concat('0',hour), hour))\" +\n      \"order by cc_no\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:03:16+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup1: org.apache.spark.sql.DataFrame = [cc_no: string, hour: int ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511217755666_1946349649","id":"20171118-224551_463420359","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:03:16+0000","dateFinished":"2017-11-21T13:03:16+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1067"},{"text":"rollup1.count()","user":"anonymous","dateUpdated":"2017-11-21T13:03:20+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres11: Long = 5\n"}]},"apps":[],"jobName":"paragraph_1511217755666_1946349649","id":"20171119-002750_1045682413","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:03:20+0000","dateFinished":"2017-11-21T13:03:21+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1068"},{"text":"rollup1.printSchema()","user":"anonymous","dateUpdated":"2017-11-21T13:03:28+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- cc_no: string (nullable = true)\n |-- hour: integer (nullable = true)\n |-- total_amount: double (nullable = true)\n |-- min_amount: double (nullable = true)\n |-- max_amount: double (nullable = true)\n |-- total_count: long (nullable = false)\n\n"}]},"apps":[],"jobName":"paragraph_1511217755666_1946349649","id":"20171120-132253_170713426","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:03:28+0000","dateFinished":"2017-11-21T13:03:28+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1069"},{"text":"rollup1.show(10)","user":"anonymous","dateUpdated":"2017-11-21T13:03:31+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+----------+------------+----------+----------+-----------+\n|           cc_no|      hour|total_amount|min_amount|max_amount|total_count|\n+----------------+----------+------------+----------+----------+-----------+\n|1234123412341234|2016030911|       200.0|     200.0|     200.0|          1|\n|1234123412341234|2016031721|       200.0|     200.0|     200.0|          1|\n|1234123412341235|2016030911|      1200.0|     400.0|     800.0|          2|\n|1234123412341236|2016030911|       750.0|     750.0|     750.0|          1|\n|1234123412341237|2016030912|      1500.0|    1500.0|    1500.0|          1|\n+----------------+----------+------------+----------+----------+-----------+\n\n"}]},"apps":[],"jobName":"paragraph_1511217755668_1944041155","id":"20171120-132504_1357441555","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:03:31+0000","dateFinished":"2017-11-21T13:03:32+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1070"},{"text":"val rollup1D= sqlContext.sql(\"select distinct (cc_no) from my_transactions\")","user":"anonymous","dateUpdated":"2017-11-21T13:03:59+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup1D: org.apache.spark.sql.DataFrame = [cc_no: string]\n"}]},"apps":[],"jobName":"paragraph_1511217755667_1945964900","id":"20171119-002949_355129101","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:03:59+0000","dateFinished":"2017-11-21T13:03:59+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1071"},{"text":"rollup1D.show","user":"anonymous","dateUpdated":"2017-11-21T13:04:03+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+\n|           cc_no|\n+----------------+\n|1234123412341234|\n|1234123412341235|\n|1234123412341236|\n|1234123412341237|\n+----------------+\n\n"}]},"apps":[],"jobName":"paragraph_1511217755667_1945964900","id":"20171119-003010_664538894","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:03+0000","dateFinished":"2017-11-21T13:04:03+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1072"},{"text":"rollup1D.count","user":"anonymous","dateUpdated":"2017-11-21T13:04:09+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres15: Long = 4\n"}]},"apps":[],"jobName":"paragraph_1511217755668_1944041155","id":"20171120-172234_672595596","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:09+0000","dateFinished":"2017-11-21T13:04:09+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1073"},{"text":"rollup1.write.format(\"org.apache.spark.sql.cassandra\")\n      .mode(SaveMode.Overwrite)\n      .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"hourlyaggregates_bycc\", \"confirm.truncate\" -> \"true\"))\n      .save()\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:11+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1511217755668_1944041155","id":"20171120-141445_1032996535","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:11+0000","dateFinished":"2017-11-21T13:04:13+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1074"},{"text":"    // 2. dailyaggregates_bycc\n    println(\" - 2. Populating dailyaggregates_bycc\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:19+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":" - 2. Populating dailyaggregates_bycc\n"}]},"apps":[],"jobName":"paragraph_1511217755669_1943656406","id":"20171118-231058_1871349931","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:19+0000","dateFinished":"2017-11-21T13:04:20+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1075"},{"text":"    val rollup2= sqlContext.sql(\"select cc_no, \" +\n      \"int(concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day))) as day, \" +\n      \"sum(amount) as total_amount, \" +\n      \"min(amount) as min_amount, \" +\n      \"max(amount) as max_amount, \" +\n      \"count(*) as total_count \" +\n      \"from my_transactions \" +\n      \"group by cc_no, \" +\n      \"concat(year, if(length(month)=1, concat('0',month), month),if(length(day)=1, concat('0',day), day))\" +\n      \"order by cc_no\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:22+0000","config":{"editorSetting":{"language":"scala","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/scala","editorHide":false,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup2: org.apache.spark.sql.DataFrame = [cc_no: string, day: int ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511217755669_1943656406","id":"20171120-151755_1320940199","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:23+0000","dateFinished":"2017-11-21T13:04:23+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1076"},{"text":"rollup2.printSchema()","user":"anonymous","dateUpdated":"2017-11-21T13:04:25+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- cc_no: string (nullable = true)\n |-- day: integer (nullable = true)\n |-- total_amount: double (nullable = true)\n |-- min_amount: double (nullable = true)\n |-- max_amount: double (nullable = true)\n |-- total_count: long (nullable = false)\n\n"}]},"apps":[],"jobName":"paragraph_1511217755670_1944810653","id":"20171119-012725_1331125911","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:25+0000","dateFinished":"2017-11-21T13:04:26+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1077"},{"text":"rollup2.show(10)","user":"anonymous","dateUpdated":"2017-11-21T13:04:33+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+--------+------------+----------+----------+-----------+\n|           cc_no|     day|total_amount|min_amount|max_amount|total_count|\n+----------------+--------+------------+----------+----------+-----------+\n|1234123412341234|20160309|       200.0|     200.0|     200.0|          1|\n|1234123412341234|20160317|       200.0|     200.0|     200.0|          1|\n|1234123412341235|20160309|      1200.0|     400.0|     800.0|          2|\n|1234123412341236|20160309|       750.0|     750.0|     750.0|          1|\n|1234123412341237|20160309|      1500.0|    1500.0|    1500.0|          1|\n+----------------+--------+------------+----------+----------+-----------+\n\n"}]},"apps":[],"jobName":"paragraph_1511217755672_1942502160","id":"20171120-151649_1133532032","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:33+0000","dateFinished":"2017-11-21T13:04:33+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1078"},{"text":"rollup2.count","user":"anonymous","dateUpdated":"2017-11-21T13:04:45+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres21: Long = 5\n"}]},"apps":[],"jobName":"paragraph_1511217755670_1944810653","id":"20171119-012848_708758351","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:45+0000","dateFinished":"2017-11-21T13:04:46+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1079"},{"text":"     rollup2.write.format(\"org.apache.spark.sql.cassandra\")\n        .mode(SaveMode.Append)\n        .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"dailyaggregates_bycc\"))\n        .save()\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:50+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1511217755672_1942502160","id":"20171118-231159_471004502","dateCreated":"2017-11-20T22:42:35+0000","dateStarted":"2017-11-21T13:04:50+0000","dateFinished":"2017-11-21T13:04:51+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1080"},{"text":"    // 3. monthlyaggregates_bycc\n    println(\" - 3. Populating monthlyaggregates_bycc\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:54+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":" - 3. Populating monthlyaggregates_bycc\n"}]},"apps":[],"jobName":"paragraph_1511266884459_-1687793870","id":"20171121-122124_1068615353","dateCreated":"2017-11-21T12:21:24+0000","dateStarted":"2017-11-21T13:04:54+0000","dateFinished":"2017-11-21T13:04:54+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1081"},{"text":"    val rollup3= sqlContext.sql(\"select cc_no, \" +\n      \"int(concat(year, if(length(month)=1, concat('0',month), month))) as month, \" +\n      \"sum(amount) as total_amount, \" +\n      \"min(amount) as min_amount, \" +\n      \"max(amount) as max_amount, \" +\n      \"count(*) as total_count \" +\n      \"from my_transactions \" +\n      \"group by cc_no, concat(year, if(length(month)=1, concat('0',month), month))\" +\n      \"order by cc_no\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:04:56+0000","config":{"editorSetting":{"language":"scala","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/scala","editorHide":false,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup3: org.apache.spark.sql.DataFrame = [cc_no: string, month: int ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511266954961_1196992086","id":"20171121-122234_1117938514","dateCreated":"2017-11-21T12:22:34+0000","dateStarted":"2017-11-21T13:04:56+0000","dateFinished":"2017-11-21T13:04:57+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1082"},{"text":"rollup3.printSchema()","user":"anonymous","dateUpdated":"2017-11-21T13:05:00+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- cc_no: string (nullable = true)\n |-- month: integer (nullable = true)\n |-- total_amount: double (nullable = true)\n |-- min_amount: double (nullable = true)\n |-- max_amount: double (nullable = true)\n |-- total_count: long (nullable = false)\n\n"}]},"apps":[],"jobName":"paragraph_1511266990445_-1593024692","id":"20171121-122310_1214914835","dateCreated":"2017-11-21T12:23:10+0000","dateStarted":"2017-11-21T13:05:00+0000","dateFinished":"2017-11-21T13:05:00+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1083"},{"text":"rollup3.show(10)","user":"anonymous","dateUpdated":"2017-11-21T13:05:03+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+------+------------+----------+----------+-----------+\n|           cc_no| month|total_amount|min_amount|max_amount|total_count|\n+----------------+------+------------+----------+----------+-----------+\n|1234123412341234|201603|       400.0|     200.0|     200.0|          2|\n|1234123412341235|201603|      1200.0|     400.0|     800.0|          2|\n|1234123412341236|201603|       750.0|     750.0|     750.0|          1|\n|1234123412341237|201603|      1500.0|    1500.0|    1500.0|          1|\n+----------------+------+------------+----------+----------+-----------+\n\n"}]},"apps":[],"jobName":"paragraph_1511267036162_-1055069108","id":"20171121-122356_339898643","dateCreated":"2017-11-21T12:23:56+0000","dateStarted":"2017-11-21T13:05:03+0000","dateFinished":"2017-11-21T13:05:03+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1084"},{"text":"     rollup3.write.format(\"org.apache.spark.sql.cassandra\")\n        .mode(SaveMode.Append)\n        .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"monthlyaggregates_bycc\"))\n        .save()\n","user":"anonymous","dateUpdated":"2017-11-21T13:05:06+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1511267062966_2093962721","id":"20171121-122422_295032929","dateCreated":"2017-11-21T12:24:22+0000","dateStarted":"2017-11-21T13:05:06+0000","dateFinished":"2017-11-21T13:05:07+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1085"},{"text":"    // 4. yearlyaggregates_bycc\n    println(\" - 4. Populating yearlyaggregates_bycc\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:05:10+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":" - 4. Populating yearlyaggregates_bycc\n"}]},"apps":[],"jobName":"paragraph_1511267269969_-1753081878","id":"20171121-122749_1417039940","dateCreated":"2017-11-21T12:27:49+0000","dateStarted":"2017-11-21T13:05:10+0000","dateFinished":"2017-11-21T13:05:10+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1086"},{"text":"    val rollup4= sqlContext.sql(\"select cc_no, \" +\n      \"int(year) as year, \" +\n      \"sum(amount) as total_amount, \" +\n      \"min(amount) as min_amount, \" +\n      \"max(amount) as max_amount, \" +\n      \"count(*) as total_count \" +\n      \"from my_transactions group by cc_no, int(year)\" +\n      \"order by cc_no\")\n","user":"anonymous","dateUpdated":"2017-11-21T13:05:12+0000","config":{"editorSetting":{"language":"scala","editOnDblClick":false},"colWidth":12,"editorMode":"ace/mode/scala","editorHide":false,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup4: org.apache.spark.sql.DataFrame = [cc_no: string, year: int ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511267287133_-396796033","id":"20171121-122807_570845439","dateCreated":"2017-11-21T12:28:07+0000","dateStarted":"2017-11-21T13:05:12+0000","dateFinished":"2017-11-21T13:05:13+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1087"},{"text":"rollup4.printSchema()","user":"anonymous","dateUpdated":"2017-11-21T13:05:15+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- cc_no: string (nullable = true)\n |-- year: integer (nullable = true)\n |-- total_amount: double (nullable = true)\n |-- min_amount: double (nullable = true)\n |-- max_amount: double (nullable = true)\n |-- total_count: long (nullable = false)\n\n"}]},"apps":[],"jobName":"paragraph_1511267308794_2001190619","id":"20171121-122828_1731698529","dateCreated":"2017-11-21T12:28:28+0000","dateStarted":"2017-11-21T13:05:15+0000","dateFinished":"2017-11-21T13:05:16+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1088"},{"text":"rollup4.show(10)","user":"anonymous","dateUpdated":"2017-11-21T13:05:18+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------------+----+------------+----------+----------+-----------+\n|           cc_no|year|total_amount|min_amount|max_amount|total_count|\n+----------------+----+------------+----------+----------+-----------+\n|1234123412341234|2016|       400.0|     200.0|     200.0|          2|\n|1234123412341235|2016|      1200.0|     400.0|     800.0|          2|\n|1234123412341236|2016|       750.0|     750.0|     750.0|          1|\n|1234123412341237|2016|      1500.0|    1500.0|    1500.0|          1|\n+----------------+----+------------+----------+----------+-----------+\n\n"}]},"apps":[],"jobName":"paragraph_1511267319926_-1237225549","id":"20171121-122839_1159627835","dateCreated":"2017-11-21T12:28:39+0000","dateStarted":"2017-11-21T13:05:19+0000","dateFinished":"2017-11-21T13:05:19+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1089"},{"text":"     rollup4.write.format(\"org.apache.spark.sql.cassandra\")\n        .mode(SaveMode.Append)\n        .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"yearlyaggregates_bycc\"))\n        .save()\n","user":"anonymous","dateUpdated":"2017-11-21T13:24:21+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1511267334749_2080864290","id":"20171121-122854_1653935049","dateCreated":"2017-11-21T12:28:54+0000","dateStarted":"2017-11-21T13:24:21+0000","dateFinished":"2017-11-21T13:24:21+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1090"},{"text":"println(\"Completed RollUps By Merchant\")\n","dateUpdated":"2017-11-21T12:39:02+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"Completed RollUps By Merchant\n"}]},"apps":[],"jobName":"paragraph_1511217755672_1942502160","id":"20171118-231215_1472031606","dateCreated":"2017-11-20T22:42:35+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:1091"},{"dateUpdated":"2017-11-20T22:42:35+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1511217755673_1942117411","id":"20171118-231241_1558161254","dateCreated":"2017-11-20T22:42:35+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:1092"}],"name":"RTFAP2 - RUP by Card","id":"2CZDJW67E","angularObjects":{"2CZ9ZVAFQ:shared_process":[],"2CY323U66:shared_process":[],"2D1DGBGW1:shared_process":[],"2D1DNX3KF:shared_process":[],"2CYC7CVWF:shared_process":[],"2CYGC3EHV:shared_process":[],"2CYM9DYY2:shared_process":[],"2D1KY1WK9:shared_process":[],"2CZDMJCR5:shared_process":[],"2CYAVWBPW:shared_process":[],"2CXS3R2CT:shared_process":[],"2CXRHDF6W:shared_process":[],"2CYGP1DAF:shared_process":[],"2CXR81BMT:shared_process":[],"2CXB21K7Y:shared_process":[],"2CZXDYFE1:shared_process":[],"2CYEDZFA4:shared_process":[],"2CXGEK2F9:shared_process":[],"2D18Y78GK:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}