-
Notifications
You must be signed in to change notification settings - Fork 29
/
Copy pathRTFAP2 - RUP by Merchant.json
1 lines (1 loc) · 31.3 KB
/
RTFAP2 - RUP by Merchant.json
1
{"paragraphs":[{"text":"import org.apache.spark.sql.hive.HiveContext\nimport org.apache.spark.{SparkConf, SparkContext}\nimport org.apache.spark.sql.{SQLContext, SaveMode}\n","user":"anonymous","dateUpdated":"2017-11-21T12:54:10+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nimport org.apache.spark.sql.hive.HiveContext\n\nimport org.apache.spark.{SparkConf, SparkContext}\n\nimport org.apache.spark.sql.{SQLContext, SaveMode}\n"}]},"apps":[],"jobName":"paragraph_1511181556172_-1354490184","id":"20171118-224353_159192803","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:11+0000","dateFinished":"2017-11-21T12:54:11+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:4623"},{"text":"val conf = new SparkConf().setAppName(\"RollUpReportsByMerchant\").set(\"spark.ui.port\", \"40400\" ).set(\"spark.driver.allowMultipleContexts\", \"true\")","user":"anonymous","dateUpdated":"2017-11-21T13:33:06+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nconf: org.apache.spark.SparkConf = org.apache.spark.SparkConf@48f56bce\n"}]},"apps":[],"jobName":"paragraph_1511203873694_1627095973","id":"20171120-185113_356402705","dateCreated":"2017-11-20T18:51:13+0000","dateStarted":"2017-11-21T12:54:17+0000","dateFinished":"2017-11-21T12:54:17+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4624"},{"text":"val sc = SparkContext.getOrCreate(conf)","user":"anonymous","dateUpdated":"2017-11-21T12:54:25+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nsc: org.apache.spark.SparkContext = org.apache.spark.SparkContext@63a1e525\n"}]},"apps":[],"jobName":"paragraph_1511181556172_-1354490184","id":"20171118-224510_926766130","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:25+0000","dateFinished":"2017-11-21T12:54:25+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4625"},{"text":"val sqlContext = new HiveContext(sc)","user":"anonymous","dateUpdated":"2017-11-21T12:54:27+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nwarning: there was one deprecation warning; re-run with -deprecation for details\n\nsqlContext: org.apache.spark.sql.hive.HiveContext = org.apache.spark.sql.hive.HiveContext@a0aed2f\n"}]},"apps":[],"jobName":"paragraph_1511181556173_-1354874932","id":"20171118-224522_1019518227","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:27+0000","dateFinished":"2017-11-21T12:54:27+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4626"},{"text":" println(\"-------- 1st , RollUp transactions by merchant by cc by day and save to dailytxns_bymerchant table --------\")\n println(\"- Populating dailytxns_bymerchant\")\n println(\"Keyed on merchant and day and clustered by time\")","user":"anonymous","dateUpdated":"2017-11-21T12:54:30+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"-------- 1st , RollUp transactions by merchant by cc by day and save to dailytxns_bymerchant table --------\n- Populating dailytxns_bymerchant\nKeyed on merchant and day and clustered by time\n"}]},"apps":[],"jobName":"paragraph_1511181556173_-1354874932","id":"20171119-013128_10870176","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:30+0000","dateFinished":"2017-11-21T12:54:30+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4627"},{"text":"sqlContext.sql(\"\"\"CREATE or replace TEMPORARY VIEW temp_transactions\n USING org.apache.spark.sql.cassandra\n OPTIONS (\n table \"transactions\",\n keyspace \"rtfap\",\n cluster \"Test Cluster\",\n pushdown \"true\"\n )\"\"\")\n","user":"anonymous","dateUpdated":"2017-11-21T12:54:32+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres95: org.apache.spark.sql.DataFrame = []\n"}]},"apps":[],"jobName":"paragraph_1511181556173_-1354874932","id":"20171118-224534_255374190","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:32+0000","dateFinished":"2017-11-21T12:54:32+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4628"},{"text":"val rollup1= sqlContext.sql(\"select merchant, \" + \n \"int(translate(string(date(txn_time)),'-','')) as day, \" +\n \"cc_no, \" +\n \"txn_id, \" +\n \"txn_time, \" +\n \"amount, \" +\n \"cc_provider, \" +\n \"items, \" +\n \"location, \" +\n \"notes, \" +\n \"status, \" +\n \"tags, \" +\n \"user_id \" + \n \"from temp_transactions\")","user":"anonymous","dateUpdated":"2017-11-21T12:54:35+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup1: org.apache.spark.sql.DataFrame = [merchant: string, day: int ... 11 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511181556173_-1354874932","id":"20171118-224551_463420359","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:35+0000","dateFinished":"2017-11-21T12:54:35+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4629"},{"text":"rollup1.count()","user":"anonymous","dateUpdated":"2017-11-21T12:54:42+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"ERROR","msg":[{"type":"TEXT","data":"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\norg.apache.spark.SparkException: Job 52 cancelled part of cancelled job group zeppelin-20171119-002750_1045682413\n at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1454)\n at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1393)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply$mcVI$sp(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply(DAGScheduler.scala:795)\n at scala.collection.mutable.HashSet.foreach(HashSet.scala:78)\n at org.apache.spark.scheduler.DAGScheduler.handleJobGroupCancelled(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1642)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1622)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1611)\n at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:632)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1873)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1886)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1899)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1913)\n at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:912)\n at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)\n at org.apache.spark.rdd.RDD.collect(RDD.scala:911)\n at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:290)\n at org.apache.spark.sql.Dataset$$anonfun$org$apache$spark$sql$Dataset$$execute$1$1.apply(Dataset.scala:2193)\n at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)\n at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2546)\n at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$execute$1(Dataset.scala:2192)\n at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collect(Dataset.scala:2199)\n at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2227)\n at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2226)\n at org.apache.spark.sql.Dataset.withCallback(Dataset.scala:2559)\n at org.apache.spark.sql.Dataset.count(Dataset.scala:2226)\n ... 46 elided\n"}]},"apps":[],"jobName":"paragraph_1511181556174_-1353720686","id":"20171119-002750_1045682413","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:54:42+0000","dateFinished":"2017-11-21T12:54:49+0000","status":"ERROR","progressUpdateIntervalMs":500,"$$hashKey":"object:4630"},{"text":"rollup1.printSchema()","user":"anonymous","dateUpdated":"2017-11-21T12:44:34+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- merchant: string (nullable = true)\n |-- day: integer (nullable = true)\n |-- cc_no: string (nullable = true)\n |-- txn_id: string (nullable = true)\n |-- txn_time: timestamp (nullable = true)\n |-- amount: double (nullable = true)\n |-- cc_provider: string (nullable = true)\n |-- items: map (nullable = true)\n | |-- key: string\n | |-- value: double (valueContainsNull = true)\n |-- location: string (nullable = true)\n |-- notes: string (nullable = true)\n |-- status: string (nullable = true)\n |-- tags: array (nullable = true)\n | |-- element: string (containsNull = true)\n |-- user_id: string (nullable = true)\n\n"}]},"apps":[],"jobName":"paragraph_1511184173723_1822919999","id":"20171120-132253_170713426","dateCreated":"2017-11-20T13:22:53+0000","dateStarted":"2017-11-21T12:44:34+0000","dateFinished":"2017-11-21T12:44:34+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4631"},{"text":"val rollup1D= sqlContext.sql(\"select distinct (merchant) from temp_transactions\")","user":"anonymous","dateUpdated":"2017-11-21T12:44:38+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup1D: org.apache.spark.sql.DataFrame = [merchant: string]\n"}]},"apps":[],"jobName":"paragraph_1511181556174_-1353720686","id":"20171119-002949_355129101","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:44:38+0000","dateFinished":"2017-11-21T12:44:38+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4632"},{"text":"rollup1D.show","user":"anonymous","dateUpdated":"2017-11-21T12:44:41+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------+\n| merchant|\n+----------+\n| Nordstrom|\n|Ann Taylor|\n| Macy|\n| Harrods|\n| GAP|\n+----------+\n\n"}]},"apps":[],"jobName":"paragraph_1511181556174_-1353720686","id":"20171119-003010_664538894","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-21T12:44:41+0000","dateFinished":"2017-11-21T12:44:41+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4633"},{"text":"rollup1D.count","user":"anonymous","dateUpdated":"2017-11-21T12:44:45+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres84: Long = 5\n"}]},"apps":[],"jobName":"paragraph_1511198554155_1237944698","id":"20171120-172234_672595596","dateCreated":"2017-11-20T17:22:34+0000","dateStarted":"2017-11-21T12:44:45+0000","dateFinished":"2017-11-21T12:44:45+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4634"},{"text":"rollup1.show(10)","user":"anonymous","dateUpdated":"2017-11-21T12:44:55+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+----------+--------+----------------+------+--------------------+------+-----------+--------------------+-------------+--------------------+--------+------------+-------------+\n| merchant| day| cc_no|txn_id| txn_time|amount|cc_provider| items| location| notes| status| tags| user_id|\n+----------+--------+----------------+------+--------------------+------+-----------+--------------------+-------------+--------------------+--------+------------+-------------+\n| Harrods|20160309|1234123412341235|982538|2016-03-09 11:04:...| 800.0| VISA|Map(chocolates ->...| London|customer likes el...|Approved| [HighValue]|simonanbridge|\n| Macy|20160309|1234123412341235|876354|2016-03-09 11:04:...| 400.0| VISA|Map(cap -> 25.0, ...| San Diego|cool stuff-good c...|Rejected|[Fraudulent]|simonanbridge|\n| GAP|20160309|1234123412341236|092753|2016-03-09 11:04:...| 750.0| MASTERCARD|Map(belts -> 150....| San Jose|customer likes el...|Approved| [HighValue]| cary|\n|Ann Taylor|20160309|1234123412341237|876302|2016-03-09 12:30:...|1500.0| AMEX|Map(clothes -> 15...| New York| frequent customer|Approved| [HighValue]| caroline|\n| Nordstrom|20160309|1234123412341234|098765|2016-03-09 11:04:...| 200.0| VISA|Map(dressshirt ->...|San Francisco|pretty good clothing|Approved|[Suspicious]| kunalak|\n| Nordstrom|20160317|1234123412341234|763629|2016-03-17 21:04:...| 200.0| VISA|Map(dressshirt ->...|San Francisco| asked for discounts|Approved|[Fraudulent]| kunalak|\n+----------+--------+----------------+------+--------------------+------+-----------+--------------------+-------------+--------------------+--------+------------+-------------+\n\n"}]},"apps":[],"jobName":"paragraph_1511184304336_888181030","id":"20171120-132504_1357441555","dateCreated":"2017-11-20T13:25:04+0000","dateStarted":"2017-11-21T12:44:55+0000","dateFinished":"2017-11-21T12:44:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4635"},{"text":" rollup1.write.format(\"org.apache.spark.sql.cassandra\")\n .mode(SaveMode.Overwrite)\n .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"dailytxns_bymerchant\", \"confirm.truncate\" -> \"true\"))\n .save()\n","user":"anonymous","dateUpdated":"2017-11-21T12:45:08+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"ERROR","msg":[{"type":"TEXT","data":"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\norg.apache.spark.SparkException: Job 50 cancelled part of cancelled job group zeppelin-20171120-141445_1032996535\n at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1454)\n at org.apache.spark.scheduler.DAGScheduler.handleJobCancellation(DAGScheduler.scala:1393)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply$mcVI$sp(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleJobGroupCancelled$1.apply(DAGScheduler.scala:795)\n at scala.collection.mutable.HashSet.foreach(HashSet.scala:78)\n at org.apache.spark.scheduler.DAGScheduler.handleJobGroupCancelled(DAGScheduler.scala:795)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1642)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1622)\n at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1611)\n at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:632)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1873)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1886)\n at org.apache.spark.SparkContext.runJob(SparkContext.scala:1906)\n at com.datastax.spark.connector.RDDFunctions.saveToCassandra(RDDFunctions.scala:36)\n at org.apache.spark.sql.cassandra.CassandraSourceRelation.insert(CassandraSourceRelation.scala:65)\n at org.apache.spark.sql.cassandra.DefaultSource.createRelation(DefaultSource.scala:87)\n at org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:457)\n at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:211)\n ... 46 elided\n"}]},"apps":[],"jobName":"paragraph_1511187285466_2077085183","id":"20171120-141445_1032996535","dateCreated":"2017-11-20T14:14:45+0000","dateStarted":"2017-11-21T12:45:08+0000","dateFinished":"2017-11-21T12:46:57+0000","status":"ERROR","progressUpdateIntervalMs":500,"$$hashKey":"object:4636"},{"text":"println(\"------- 2nd, Do the aggregations for daily totals etc., Save to Static columns in the dailytxns_bymerchant table -------\")\nprintln(\"- Aggregating in dailytxns_bymerchant\")\n","user":"anonymous","dateUpdated":"2017-11-20T17:55:21+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"------- 2nd, Do the aggregations for daily totals etc., Save to Static columns in the dailytxns_bymerchant table -------\n- Aggregating in dailytxns_bymerchant\n"}]},"apps":[],"jobName":"paragraph_1511181556177_-1344101963","id":"20171118-231058_1871349931","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:21+0000","dateFinished":"2017-11-20T17:55:21+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4637"},{"text":"sqlContext.sql(\"drop view temp_dailytxns_bymerchant\");","user":"anonymous","dateUpdated":"2017-11-20T17:55:25+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres146: org.apache.spark.sql.DataFrame = []\n"}]},"apps":[],"jobName":"paragraph_1511181556178_-1342947717","id":"20171119-013641_125195863","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:25+0000","dateFinished":"2017-11-20T17:55:25+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4638"},{"text":"sqlContext.sql(\"\"\"CREATE or REPLACE TEMPORARY VIEW temp_dailytxns_bymerchant\n USING org.apache.spark.sql.cassandra\n OPTIONS (\n table \"dailytxns_bymerchant\",\n keyspace \"rtfap\",\n cluster \"Test Cluster\",\n pushdown \"true\"\n )\"\"\")","user":"anonymous","dateUpdated":"2017-11-21T13:33:06+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres147: org.apache.spark.sql.DataFrame = []\n"}]},"apps":[],"jobName":"paragraph_1511181556179_-1343332465","id":"20171118-231120_1119359604","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:28+0000","dateFinished":"2017-11-20T17:55:28+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4639"},{"text":" sqlContext.udf.register(\"now\", () => System.currentTimeMillis)","user":"anonymous","dateUpdated":"2017-11-20T17:55:32+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres148: org.apache.spark.sql.expressions.UserDefinedFunction = UserDefinedFunction(<function0>,LongType,Some(List()))\n"}]},"apps":[],"jobName":"paragraph_1511181556182_-1344486712","id":"20171119-013707_2073131461","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:32+0000","dateFinished":"2017-11-20T17:55:32+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4640"},{"text":"val rollup2= sqlContext.sql(\"select merchant, \" +\n \"day, \" +\n \"count(*) as total_count, \" +\n \"sum(amount) as total_amount, \" +\n \"min(amount) as min_amount, \" +\n \"max(amount) as max_amount \" +\n \"from temp_dailytxns_bymerchant \" +\n \"group by merchant, day order by merchant\")\n","user":"anonymous","dateUpdated":"2017-11-20T17:55:36+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup2: org.apache.spark.sql.DataFrame = [merchant: string, day: int ... 4 more fields]\n"}]},"apps":[],"jobName":"paragraph_1511191075471_-374646259","id":"20171120-151755_1320940199","dateCreated":"2017-11-20T15:17:55+0000","dateStarted":"2017-11-20T17:55:36+0000","dateFinished":"2017-11-20T17:55:36+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4641"},{"text":"rollup2.count","user":"anonymous","dateUpdated":"2017-11-20T17:55:39+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nres149: Long = 100\n"}]},"apps":[],"jobName":"paragraph_1511181556183_-1344871461","id":"20171119-012848_708758351","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:39+0000","dateFinished":"2017-11-20T17:55:40+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4642"},{"text":"rollup2.printSchema()","user":"anonymous","dateUpdated":"2017-11-20T17:55:43+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"root\n |-- merchant: string (nullable = true)\n |-- day: integer (nullable = true)\n |-- total_count: long (nullable = false)\n |-- total_amount: double (nullable = true)\n |-- min_amount: double (nullable = true)\n |-- max_amount: double (nullable = true)\n\n"}]},"apps":[],"jobName":"paragraph_1511181556182_-1344486712","id":"20171119-012725_1331125911","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:55:43+0000","dateFinished":"2017-11-20T17:55:43+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4643"},{"text":"val rollup2D= sqlContext.sql(\"select count (distinct cc_no) from temp_dailytxns_bymerchant\")","user":"anonymous","dateUpdated":"2017-11-20T17:55:45+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup2D: org.apache.spark.sql.DataFrame = [count(DISTINCT cc_no): bigint]\n"}]},"apps":[],"jobName":"paragraph_1511195348673_-589975424","id":"20171120-162908_1910996562","dateCreated":"2017-11-20T16:29:08+0000","dateStarted":"2017-11-20T17:55:45+0000","dateFinished":"2017-11-20T17:55:46+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4644"},{"text":"rollup2D.show","user":"anonymous","dateUpdated":"2017-11-20T17:55:48+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+---------------------+\n|count(DISTINCT cc_no)|\n+---------------------+\n| 3391|\n+---------------------+\n\n"}]},"apps":[],"jobName":"paragraph_1511195385416_1211464899","id":"20171120-162945_1260554465","dateCreated":"2017-11-20T16:29:45+0000","dateStarted":"2017-11-20T17:55:48+0000","dateFinished":"2017-11-20T17:55:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4645"},{"text":"val rollup2D= sqlContext.sql(\"select count (distinct merchant) from temp_dailytxns_bymerchant\")","user":"anonymous","dateUpdated":"2017-11-20T17:55:52+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"\nrollup2D: org.apache.spark.sql.DataFrame = [count(DISTINCT merchant): bigint]\n"}]},"apps":[],"jobName":"paragraph_1511199728081_-1082235141","id":"20171120-174208_1010960965","dateCreated":"2017-11-20T17:42:08+0000","dateStarted":"2017-11-20T17:55:52+0000","dateFinished":"2017-11-20T17:55:52+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4646"},{"text":"rollup2D.show","user":"anonymous","dateUpdated":"2017-11-20T17:55:54+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false},"editorHide":false},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------------+\n|count(DISTINCT merchant)|\n+------------------------+\n| 100|\n+------------------------+\n\n"}]},"apps":[],"jobName":"paragraph_1511199740351_535465418","id":"20171120-174220_937960713","dateCreated":"2017-11-20T17:42:20+0000","dateStarted":"2017-11-20T17:55:54+0000","dateFinished":"2017-11-20T17:55:55+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4647"},{"text":"rollup2.show(10)","user":"anonymous","dateUpdated":"2017-11-20T17:55:58+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+--------+-----------+------------------+----------+----------+\n| merchant| day|total_count| total_amount|min_amount|max_amount|\n+--------------------+--------+-----------+------------------+----------+----------+\n| 7-Eleven|20171120| 19| 22821.57| 289.24| 2354.88|\n| A&P|20171120| 11|15988.400000000001| 284.77| 2346.17|\n| AT&T Wireless|20171120| 19| 23333.36| 363.93| 2475.61|\n| AVB Brandsource|20171120| 11|11788.580000000002| 446.04| 1811.29|\n| Ace Hardware|20171120| 27| 39271.62999999999| 194.45| 3534.66|\n| Advance Auto Parts|20171120| 8| 10046.94| 754.46| 1650.17|\n| Albertsons|20171120| 38| 47159.19999999999| 100.01| 2728.36|\n| Aldi|20171120| 18|16459.649999999998| 5.51| 2541.94|\n|Alimentation Couc...|20171120| 12|16977.579999999998| 169.04| 2481.35|\n| Amazon.com|20171120| 94|126734.29999999997| 28.01| 3005.7|\n+--------------------+--------+-----------+------------------+----------+----------+\nonly showing top 10 rows\n\n"}]},"apps":[],"jobName":"paragraph_1511191009813_139470217","id":"20171120-151649_1133532032","dateCreated":"2017-11-20T15:16:49+0000","dateStarted":"2017-11-20T17:55:58+0000","dateFinished":"2017-11-20T17:55:58+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4648"},{"text":" rollup2.write.format(\"org.apache.spark.sql.cassandra\")\n .mode(SaveMode.Append)\n .options(Map(\"keyspace\" -> \"rtfap\", \"table\" -> \"dailytxns_bymerchant\"))\n .save()\n","user":"anonymous","dateUpdated":"2017-11-20T17:56:01+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1511181556183_-1344871461","id":"20171118-231159_471004502","dateCreated":"2017-11-20T12:39:16+0000","dateStarted":"2017-11-20T17:56:01+0000","dateFinished":"2017-11-20T17:56:02+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:4649"},{"text":" println(\"Completed RollUps By Merchant\")\n","dateUpdated":"2017-11-20T12:39:16+0000","config":{"colWidth":12,"editorMode":"ace/mode/scala","results":{},"enabled":true,"editorSetting":{"language":"scala","editOnDblClick":false}},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"Completed RollUps By Merchant\n"}]},"apps":[],"jobName":"paragraph_1511181556183_-1344871461","id":"20171118-231215_1472031606","dateCreated":"2017-11-20T12:39:16+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:4650"},{"dateUpdated":"2017-11-20T12:39:16+0000","config":{"colWidth":12,"enabled":true,"results":{},"editorSetting":{"language":"scala","editOnDblClick":false},"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1511181556183_-1344871461","id":"20171118-231241_1558161254","dateCreated":"2017-11-20T12:39:16+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:4651"}],"name":"RTFAP2 - RUP by Merchant","id":"2D12GX13W","angularObjects":{"2CZ9ZVAFQ:shared_process":[],"2CY323U66:shared_process":[],"2D1DGBGW1:shared_process":[],"2D1DNX3KF:shared_process":[],"2CYC7CVWF:shared_process":[],"2CYGC3EHV:shared_process":[],"2CYM9DYY2:shared_process":[],"2D1KY1WK9:shared_process":[],"2CZDMJCR5:shared_process":[],"2CYAVWBPW:shared_process":[],"2CXS3R2CT:shared_process":[],"2CXRHDF6W:shared_process":[],"2CYGP1DAF:shared_process":[],"2CXR81BMT:shared_process":[],"2CXB21K7Y:shared_process":[],"2CZXDYFE1:shared_process":[],"2CYEDZFA4:shared_process":[],"2CXGEK2F9:shared_process":[],"2D18Y78GK:shared_process":[]},"config":{"looknfeel":"default","personalizedMode":"false"},"info":{}}