Chore/makefile

Makefile

@@ -16,17 +16,22 @@ test-with-reports:
 lint:
 	venv/bin/pylint casper/
 
-run-rand:
-	venv/bin/python casper.py rand
+MODE?=rand
 
-run-rrob:
-	venv/bin/python casper.py rrob
+run-binary:
+	venv/bin/python casper.py $(MODE) --protocol binary
 
-run-full:
-	venv/bin/python casper.py full
+run-blockchain:
+	venv/bin/python casper.py $(MODE) --protocol blockchain
 
-run-nofinal:
-	venv/bin/python casper.py nofinal
+run-concurrent:
+	venv/bin/python casper.py $(MODE) --protocol concurrent
 
-run-binary:
-	venv/bin/python casper.py rand --protocol binary --report-interval 3
+run-integer:
+	venv/bin/python casper.py $(MODE) --protocol integer
+
+run-order:
+	venv/bin/python casper.py $(MODE) --protocol order
+
+run-sharding:
+	venv/bin/python casper.py $(MODE) --protocol sharding

README.md

@@ -7,7 +7,7 @@ Branch    | Tests
 master    | [![CircleCI](https://circleci.com/gh/ethereum/cbc-casper/tree/master.svg?style=svg&circle-token=071bd3b625fc240222d1add41dc796ec8bee077d)](https://circleci.com/gh/ethereum/cbc-casper/tree/master)
 develop   | [![CircleCI](https://circleci.com/gh/ethereum/cbc-casper/tree/develop.svg?style=svg&circle-token=071bd3b625fc240222d1add41dc796ec8bee077d)](https://circleci.com/gh/ethereum/cbc-casper/tree/develop)
 
-A python implementation of a class of ["correct-by-construction" consensus protocols](https://github.com/ethereum/research/tree/master/papers/cbc-consensus). Currently, this includes Casper the Friendly Ghost (a blockchain consensus protocol) and Casper the Friendly Binary Consensus Protocol. 
+A python implementation of a class of ["correct-by-construction" consensus protocols](https://github.com/ethereum/research/tree/master/papers/cbc-consensus). Currently, this includes Casper the Friendly Ghost (a blockchain consensus protocol) and Casper the Friendly Binary Consensus Protocol.
 
 Specifications for these protocols can be found here [here](https://github.com/ethereum/research/tree/master/papers/CasperTFG), but the implementation and the spec may deviate from the spec, as they are still moving targets.
 
@@ -39,21 +39,28 @@ pip install using `requirements.txt`
 ### Standard
 Standard simulations are marked up for use as follows:
 
+```
+make run-[binary | blockchain | concurrent | integer | order | sharding]
+```
+
+Each protocol represents consensus on a different data structure.
+
+Optionally, you can add a message passing mode to each protocol.
 
 ```
-make run-[rand | rrob | full | nofinal | binary]
+make run-binary MODE=rrob
 ```
 
-`rand:` each round, some randomly selected validators propagate their most recent message to other randomly selected validators, who then create new messages.
+There are currently four message passing modes:
+
+`rand:` each round, randomly selected validators propagate their most recent message to other randomly selected validators, who then create new messages.
 
 `rrob:` each round, the creator of the last round's block sends it to the next receiver, who then creates a block.
 
 `full:` each round, all validators receive all other validators previous messages, and then all create messages.
 
 `nofinal:` each round, two simultaneous round-robin message propagations occur at the same time. This results in validators never being able to finalize later blocks (they may finalize initial blocks, depending on weight distribution).
 
-`binary:` unlike the above message propagation schemes, this changes the protocol to cbc-casper with binary data structures! Instead of a blockchain, this protocol just comes to consensus on a single bit.
-
 By default, a gif and associated images of the simulation will be saved in `graphs/graph_num_0/`. These settings can be modified, along with the number of validators, the number of messages that propagate per round, and the report interval in the `config.ini`.
 
 ### Advanced

casper/protocols/sharding/block.py

@@ -0,0 +1,63 @@
+"""The block module implements the message data structure for a sharded blockchain"""
+from casper.message import Message
+NUM_MERGE_SHARDS = 2
+
+
+class Block(Message):
+    """Message data structure for a sharded blockchain"""
+
+    @classmethod
+    def is_valid_estimate(cls, estimate):
+        for key in ['prev_blocks', 'shard_ids']:
+            if key not in estimate:
+                return False
+            if not isinstance(estimate[key], set):
+                return False
+        return True
+
+    def on_shard(self, shard_id):
+        return shard_id in self.estimate['shard_ids']
+
+    def prev_block(self, shard_id):
+        """Returns the previous block on the shard: shard_id
+        Throws a KeyError if there is no previous block"""
+        if shard_id not in self.estimate['shard_ids']:
+            raise KeyError("No previous block on that shard")
+
+        for block in self.estimate['prev_blocks']:
+            # if this block is the genesis, previous is None
+            if block is None:
+                return None
+
+            # otherwise, return the previous block on that shard
+            if block.on_shard(shard_id):
+                return block
+
+        raise KeyError("Block on {}, but has no previous block on that shard!".format(shard_id))
+
+    @property
+    def is_merge_block(self):
+        return len(self.estimate['shard_ids']) == NUM_MERGE_SHARDS
+
+    @property
+    def is_genesis_block(self):
+        return None in self.estimate['prev_blocks']
+
+    def conflicts_with(self, message):
+        """Returns true if self is not in the prev blocks of other_message"""
+        assert isinstance(message, Block), "...expected a block"
+
+        return not self.is_in_blockchain(message, '')
+
+    def is_in_blockchain(self, block, shard_id):
+        """Could be a zero generation ancestor!"""
+        if not block:
+            return False
+
+        if not block.on_shard(shard_id):
+            return False
+
+        if self == block:
+            return True
+
+        return self.is_in_blockchain(block.prev_block(shard_id), shard_id)

casper/protocols/sharding/forkchoice.py

@@ -0,0 +1,88 @@
+"""The forkchoice module implements the estimator function a sharded blockchain"""
+
+
+def get_max_weight_indexes(scores):
+    """Returns the keys that map to the max value in a dict.
+    The max value must be greater than zero."""
+
+    max_score = max(scores.values())
+
+    assert max_score > 0, "max_score should be greater than zero"
+
+    max_weight_estimates = {e for e in scores if scores[e] == max_score}
+
+    return max_weight_estimates
+
+
+def get_scores(starting_block, latest_messages, shard_id):
+    """Returns a dict of block => weight"""
+    scores = dict()
+
+    for validator, current_block in latest_messages.items():
+        if not current_block.on_shard(shard_id):
+            continue
+
+        while current_block and current_block != starting_block:
+            scores[current_block] = scores.get(current_block, 0) + validator.weight
+            current_block = current_block.prev_block(shard_id)
+
+    return scores
+
+
+def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
+    """Get the forkchoice for a specific shard"""
+
+    scores = get_scores(starting_block, latest_messages, shard_id)
+
+    best_block = starting_block
+    while best_block in children:
+        curr_scores = dict()
+        max_score = 0
+        for child in children[best_block]:
+            if not child.on_shard(shard_id):
+                continue  # we only select children on the same shard
+            # can't pick a child that a merge block with a higher shard
+            if child.is_merge_block:
+                not_in_forkchoice = False
+                for shard in child.estimate['shard_ids']:
+                    if len(shard) < len(shard_id):
+                        not_in_forkchoice = True
+                        break
+                if not_in_forkchoice:
+                    continue
+            curr_scores[child] = scores.get(child, 0)
+            max_score = max(curr_scores[child], max_score)
+
+        # If no child on shard, or 0 weight block, stop
+        if max_score == 0:
+            break
+
+        max_weight_children = get_max_weight_indexes(curr_scores)
+
+        assert len(max_weight_children) == 1, "... there should be no ties!"
+
+        best_block = max_weight_children.pop()
+
+    return best_block
+
+
+def get_all_shards_fork_choice(starting_blocks, children, latest_messages_on_shard):
+    """Returns a dict of shard_id -> forkchoice.
+    Starts from starting block for shard, and stops when it reaches tip"""
+
+    # for any shard we have latest messages on, we should have a starting block
+    for key in starting_blocks.keys():
+        assert key in latest_messages_on_shard
+    for key in latest_messages_on_shard.keys():
+        assert key in latest_messages_on_shard
+
+    shards_forkchoice = {
+        shard_id: get_shard_fork_choice(
+            starting_blocks[shard_id],
+            children,
+            latest_messages_on_shard[shard_id],
+            shard_id
+        ) for shard_id in starting_blocks
+    }
+
+    return shards_forkchoice

casper/protocols/sharding/sharding_plot_tool.py

@@ -0,0 +1,105 @@
+"""The blockchain plot tool implements functions for plotting sharded blockchain data structures"""
+
+from casper.plot_tool import PlotTool
+from casper.safety_oracles.clique_oracle import CliqueOracle
+import casper.utils as utils
+
+
+class ShardingPlotTool(PlotTool):
+    """The module contains functions for plotting a blockchain data structure"""
+
+    def __init__(self, display, save, view, validator_set):
+        super().__init__(display, save, 's')
+        self.view = view
+        self.validator_set = validator_set
+        self.starting_blocks = self.view.starting_blocks
+        self.message_fault_tolerance = dict()
+
+        self.blockchain = []
+        self.communications = []
+
+        self.block_fault_tolerance = {}
+        self.message_labels = {}
+        self.justifications = {
+            validator: []
+            for validator in validator_set
+        }
+
+    def update(self, new_messages=None):
+        """Updates displayable items with new messages and paths"""
+        return
+
+        if new_messages is None:
+            new_messages = []
+
+        self._update_new_justifications(new_messages)
+        self._update_blockchain(new_messages)
+        self._update_block_fault_tolerance()
+        self._update_message_labels(new_messages)
+
+    def plot(self):
+        """Builds relevant edges to display and creates next viewgraph using them"""
+        return
+        best_chain_edge = self.get_best_chain()
+
+        validator_chain_edges = self.get_validator_chains()
+
+        edgelist = []
+        edgelist.append(utils.edge(self.blockchain, 2, 'grey', 'solid'))
+        edgelist.append(utils.edge(self.communications, 1, 'black', 'dotted'))
+        edgelist.append(best_chain_edge)
+        edgelist.extend(validator_chain_edges)
+
+        self.next_viewgraph(
+            self.view,
+            self.validator_set,
+            edges=edgelist,
+            message_colors=self.block_fault_tolerance,
+            message_labels=self.message_labels
+        )
+
+    def get_best_chain(self):
+        """Returns an edge made of the global forkchoice to genesis"""
+        best_message = self.view.estimate()
+        best_chain = utils.build_chain(best_message, None)[:-1]
+        return utils.edge(best_chain, 5, 'red', 'solid')
+
+    def get_validator_chains(self):
+        """Returns a list of edges main from validators current forkchoice to genesis"""
+        vals_chain_edges = []
+        for validator in self.validator_set:
+            chain = utils.build_chain(validator.my_latest_message(), None)[:-1]
+            vals_chain_edges.append(utils.edge(chain, 2, 'blue', 'solid'))
+
+        return vals_chain_edges
+
+    def _update_new_justifications(self, new_messages):
+        for message in new_messages:
+            sender = message.sender
+            for validator in message.justification:
+                last_message = self.view.justified_messages[message.justification[validator]]
+                # only show if new justification
+                if last_message not in self.justifications[sender]:
+                    self.communications.append([last_message, message])
+                    self.justifications[sender].append(last_message)
+
+    def _update_blockchain(self, new_messages):
+        for message in new_messages:
+            if message.estimate is not None:
+                self.blockchain.append([message, message.estimate])
+
+    def _update_message_labels(self, new_messages):
+        for message in new_messages:
+            self.message_labels[message] = message.sequence_number
+
+    def _update_block_fault_tolerance(self):
+        tip = self.view.estimate()
+
+        while tip and self.block_fault_tolerance.get(tip, 0) != len(self.validator_set) - 1:
+            oracle = CliqueOracle(tip, self.view, self.validator_set)
+            fault_tolerance, num_node_ft = oracle.check_estimate_safety()
+
+            if fault_tolerance > 0:
+                self.block_fault_tolerance[tip] = num_node_ft
+
+            tip = tip.estimate

casper/protocols/sharding/sharding_protocol.py

@@ -0,0 +1,56 @@
+from casper.protocols.sharding.sharding_view import ShardingView
+from casper.protocols.sharding.block import Block
+from casper.protocols.sharding.sharding_plot_tool import ShardingPlotTool
+from casper.protocol import Protocol
+
+
+class ShardingProtocol(Protocol):
+    View = ShardingView
+    Message = Block
+    PlotTool = ShardingPlotTool
+
+    shard_genesis_blocks = dict()
+    curr_shard_idx = 0
+    curr_shard_ids = ['']
+
+    """Shard ID's look like this:
+           ''
+         /    \
+       '0'    '1'
+      /  \    /  \
+    '00''01''10''11'
+
+
+     Blocks can be merge mined between shards if
+     there is an edge between shards
+     That is, for ids shard_1 and shard_2, there can be a merge block if
+     abs(len(shard_1) - len(shard_2)) = 1 AND
+     for i in range(min(len(shard_1), len(shard_2))):
+        shard_1[i] = shard_2[i]
+    """
+
+    @classmethod
+    def initial_message(cls, validator):
+        """Returns a starting block for a shard"""
+        shard_id = cls.get_next_shard_id()
+
+        estimate = {'prev_blocks': set([None]), 'shard_ids': set([shard_id])}
+        cls.shard_genesis_blocks[shard_id] = Block(estimate, dict(), validator, -1, 0)
+
+        return cls.shard_genesis_blocks['']
+
+    @classmethod
+    def get_next_shard_id(cls):
+        next_id = cls.curr_shard_ids[cls.curr_shard_idx]
+        cls.curr_shard_idx += 1
+
+        if cls.curr_shard_idx == len(cls.curr_shard_ids):
+            next_ids = []
+            for shard_id in cls.curr_shard_ids:
+                next_ids.append(shard_id + '0')
+                next_ids.append(shard_id + '1')
+
+            cls.curr_shard_idx = 0
+            cls.curr_shard_ids = next_ids
+
+        return next_id