Skip to content

Commit

Permalink
More work on ReST UI
Browse files Browse the repository at this point in the history
  • Loading branch information
@simonambridge
simonambridge committed Dec 5, 2017
1 parent 24d5497 commit 644247e
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 13 deletions.
29 changes: 29 additions & 0 deletions TransactionHandlers/kafka_start.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
##################
# Zoopkeeper and Kafka demo start script
# V0.1 1/12/17
#################
echo "\$KAFKA_HOME=/usr/local/kafka/kafka_2.11-1.0.0"
cd $KAFKA_HOME
echo "Start Zookeeper"
sudo -b nohup $KAFKA_HOME/bin/zookeeper-server-start.sh $KAFKA_HOME/config/zookeeper.properties > $KAFKA_HOME/logs/nohupz.out 2>&1
echo ""
echo "Start Kafka"
sudo -b nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties > $KAFKA_HOME/logs/nohupk.out 2>&1
echo ""
echo "Create New Topic:"
$KAFKA_HOME/bin/kafka-topics.sh --zookeeper localhost:2181 --create --replication-factor 1 --partitions 1 --topic NewTransactions
echo ""
echo "Confirm that we created it correctly:"
$KAFKA_HOME/bin/kafka-topics.sh --zookeeper localhost:2181 --list
echo ""
echo "Set msg. retention period to 1 hour (360000 milliseconds)"
$KAFKA_HOME/bin/kafka-configs.sh --zookeeper localhost:2181 --entity-type topics --alter --add-config retention.ms=3600000 --entity-name NewTransactions
echo ""
echo "Display topic configuration details:"
$KAFKA_HOME/bin/kafka-configs.sh --zookeeper localhost:2181 --describe --entity-name NewTransactions --entity-type topics
echo ""
echo "Describe the 'NewTransactions' Topic:"
$KAFKA_HOME/bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic NewTransactions
echo ""
echo "List messages from beginning:"
$KAFKA_HOME/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic NewTransactions --from-beginning
18 changes: 10 additions & 8 deletions creates_and_inserts.cql
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//
// PLEASE NOTE: The Keyspace creation command will need to reflect the topology of your
// environment. As presented it will only work in a single data center test deployment.
// environment. SimpleStrategy is only intended to be used in a single data center test deployment.

CREATE KEYSPACE if not exists rtfap
WITH durable_writes = true
Expand Down Expand Up @@ -45,7 +45,7 @@ CREATE TABLE if not exists rtfap.hourlyaggregates_bycc(
min_amount double,
total_count bigint,
PRIMARY KEY ((cc_no, hour))
);
) WITH default_time_to_live = 86400;

DROP TABLE IF EXISTS rtfap.dailyaggregates_bycc;
CREATE TABLE if not exists rtfap.dailyaggregates_bycc(
Expand All @@ -56,7 +56,7 @@ CREATE TABLE if not exists rtfap.dailyaggregates_bycc(
min_amount double,
total_count bigint,
PRIMARY KEY ((cc_no, day))
);
) WITH default_time_to_live = 86400;

DROP TABLE IF EXISTS rtfap.monthlyaggregates_bycc;
CREATE TABLE if not exists rtfap.monthlyaggregates_bycc(
Expand All @@ -67,7 +67,7 @@ CREATE TABLE if not exists rtfap.monthlyaggregates_bycc(
min_amount double,
total_count bigint,
PRIMARY KEY ((cc_no, month))
);
) WITH default_time_to_live = 86400;

DROP TABLE IF EXISTS rtfap.yearlyaggregates_bycc;
CREATE TABLE if not exists rtfap.yearlyaggregates_bycc(
Expand All @@ -78,7 +78,7 @@ CREATE TABLE if not exists rtfap.yearlyaggregates_bycc(
min_amount double,
total_count bigint,
PRIMARY KEY ((cc_no, year))
);
) WITH default_time_to_live = 86400;

DROP TABLE IF EXISTS rtfap.dailytxns_bymerchant;
CREATE TABLE rtfap.dailytxns_bymerchant (
Expand All @@ -100,7 +100,7 @@ CREATE TABLE rtfap.dailytxns_bymerchant (
tags set<text>,
user_id text,
PRIMARY KEY ((merchant, day), cc_no)
) WITH CLUSTERING ORDER BY (cc_no ASC);
) WITH CLUSTERING ORDER BY (cc_no ASC) AND default_time_to_live = 86400;

DROP TABLE IF EXISTS rtfap.txn_count_min;
CREATE TABLE if not exists rtfap.txn_count_min (
Expand All @@ -117,10 +117,11 @@ CREATE TABLE if not exists rtfap.txn_count_min (
ttl_txn_hr int,
ttl_txn_min int,
PRIMARY KEY (( year, month, day, hour, minute ))
);
) WITH default_time_to_live = 86400;

// Cassandra stress table

DROP TABLE IF EXISTS rtfap.txn_by_cc;
CREATE TABLE if not exists rtfap.txn_by_cc (
cc_no text,
txn_year int,
Expand Down Expand Up @@ -149,7 +150,8 @@ AND replication = {
'replication_factor' : 1
};

CREATE TABLE banana.dashboards (id text PRIMARY KEY,
DROP TABLE IF EXISTS banana.dashboards;
CREATE TABLE if not exists banana.dashboards (id text PRIMARY KEY,
"_version_" bigint, dashboard text, group text, solr_query text, title text, user text);


Expand Down
Binary file added restRTFAP/public/.DS_Store
View file
Binary file not shown.
10 changes: 5 additions & 5 deletions restRTFAP/public/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,14 +266,14 @@ <h1>Spark & Querying Roll Up Tables</h1>

<p>This tight integration between Cassandra and Spark offers huge value in terms of significantly reduced ETL complexity (no data movement to different clusters) and thus reducing time to insight from your data through a much less complex "cohesive lambda architecture"</p>

<p>Streaming Analytics
<p><h2>Streaming Analytics</h2></p>

<p>The streaming analytics element of this application is made up of two parts:

<p>A transaction "producer" - a Scala/Akka app that generates random credit card transactions and then places those transactions onto a Kafka queue.</p>
<p>A transaction "consumer" - also written in Scala, is a Spark streaming job that (a) consumes the messages put on the Kafka queue, and then (b) parses those messages, evalutes the transaction status and then writes them to the Datastax/Cassandra table transactions.</p>
</p>It also generates rolling summary lines into the txn_count_min table every minute.</p>
<p>Streaming analytics code can be found under the directory TransactionHandlers/producer (pre-requisite: make sure you have run the CQL schema create script as described above to create the necessary tables).</p>
<p>A transaction <em>"producer"</em> - a <em2>Scala/Akka</em2> app that generates random credit card transactions and then places those transactions onto a Kafka queue.</p>
<p>A transaction <em>"consumer"</em> - also written in <em2>Scala</em2>, is a <gr>Spark</gr> streaming job that (a) consumes the messages put on the Kafka queue, and then (b) parses those messages, evalutes the transaction status and then writes them to the Datastax/Cassandra table transactions.</p>
</p>It also generates <em>rolling summary lines</em> into the txn_count_min table every minute.</p>
<p><em>Streaming analytics code</em> can be found under the directory <gr>TransactionHandlers/producer</gr> (pre-requisite: make sure you have run the CQL schema create script as described above to create the necessary tables).</p>

<p>Follow the Spark streaming installation and set up instructions <a href="https://github.com/simonambridge/RTFAP2/tree/master/TransactionHandlers/README.md">here</a> on Github</p>

Expand Down

0 comments on commit 644247e

Please sign in to comment.