Merge pull request #169 from ethereum/feat/sharding

Add blockchain sharding

Author: Nate Rush

Makefile

@@ -30,3 +30,6 @@ run-nofinal:
 
 run-binary:
 	venv/bin/python casper.py rand --protocol binary --report-interval 3
+
+run-sharding:
+	venv/bin/python casper.py rand --protocol sharding --validators 14 --report-interval 3

casper/protocols/sharding/__init__.py

@@ -0,0 +1,63 @@
+"""The block module implements the message data structure for a sharded blockchain"""
+from casper.message import Message
+NUM_MERGE_SHARDS = 2
+
+
+class Block(Message):
+    """Message data structure for a sharded blockchain"""
+
+    @classmethod
+    def is_valid_estimate(cls, estimate):
+        for key in ['prev_blocks', 'shard_ids']:
+            if key not in estimate:
+                return False
+            if not isinstance(estimate[key], set):
+                return False
+        return True
+
+    def on_shard(self, shard_id):
+        return shard_id in self.estimate['shard_ids']
+
+    def prev_block(self, shard_id):
+        """Returns the previous block on the shard: shard_id
+        Throws a KeyError if there is no previous block"""
+        if shard_id not in self.estimate['shard_ids']:
+            raise KeyError("No previous block on that shard")
+
+        for block in self.estimate['prev_blocks']:
+            # if this block is the genesis, previous is None
+            if block is None:
+                return None
+
+            # otherwise, return the previous block on that shard
+            if block.on_shard(shard_id):
+                return block
+
+        raise KeyError("Block on {}, but has no previous block on that shard!".format(shard_id))
+
+    @property
+    def is_merge_block(self):
+        return len(self.estimate['shard_ids']) == NUM_MERGE_SHARDS
+
+    @property
+    def is_genesis_block(self):
+        return None in self.estimate['prev_blocks']
+
+    def conflicts_with(self, message):
+        """Returns true if self is not in the prev blocks of other_message"""
+        assert isinstance(message, Block), "...expected a block"
+
+        return not self.is_in_blockchain(message, '')
+
+    def is_in_blockchain(self, block, shard_id):
+        """Could be a zero generation ancestor!"""
+        if not block:
+            return False
+
+        if not block.on_shard(shard_id):
+            return False
+
+        if self == block:
+            return True
+
+        return self.is_in_blockchain(block.prev_block(shard_id), shard_id)

casper/protocols/sharding/block.py

@@ -0,0 +1,88 @@
+"""The forkchoice module implements the estimator function a sharded blockchain"""
+
+
+def get_max_weight_indexes(scores):
+    """Returns the keys that map to the max value in a dict.
+    The max value must be greater than zero."""
+
+    max_score = max(scores.values())
+
+    assert max_score > 0, "max_score should be greater than zero"
+
+    max_weight_estimates = {e for e in scores if scores[e] == max_score}
+
+    return max_weight_estimates
+
+
+def get_scores(starting_block, latest_messages, shard_id):
+    """Returns a dict of block => weight"""
+    scores = dict()
+
+    for validator, current_block in latest_messages.items():
+        if not current_block.on_shard(shard_id):
+            continue
+
+        while current_block and current_block != starting_block:
+            scores[current_block] = scores.get(current_block, 0) + validator.weight
+            current_block = current_block.prev_block(shard_id)
+
+    return scores
+
+
+def get_shard_fork_choice(starting_block, children, latest_messages, shard_id):
+    """Get the forkchoice for a specific shard"""
+
+    scores = get_scores(starting_block, latest_messages, shard_id)
+
+    best_block = starting_block
+    while best_block in children:
+        curr_scores = dict()
+        max_score = 0
+        for child in children[best_block]:
+            if not child.on_shard(shard_id):
+                continue  # we only select children on the same shard
+            # can't pick a child that a merge block with a higher shard
+            if child.is_merge_block:
+                not_in_forkchoice = False
+                for shard in child.estimate['shard_ids']:
+                    if len(shard) < len(shard_id):
+                        not_in_forkchoice = True
+                        break
+                if not_in_forkchoice:
+                    continue
+            curr_scores[child] = scores.get(child, 0)
+            max_score = max(curr_scores[child], max_score)
+
+        # If no child on shard, or 0 weight block, stop
+        if max_score == 0:
+            break
+
+        max_weight_children = get_max_weight_indexes(curr_scores)
+
+        assert len(max_weight_children) == 1, "... there should be no ties!"
+
+        best_block = max_weight_children.pop()
+
+    return best_block
+
+
+def get_all_shards_fork_choice(starting_blocks, children, latest_messages_on_shard):
+    """Returns a dict of shard_id -> forkchoice.
+    Starts from starting block for shard, and stops when it reaches tip"""
+
+    # for any shard we have latest messages on, we should have a starting block
+    for key in starting_blocks.keys():
+        assert key in latest_messages_on_shard
+    for key in latest_messages_on_shard.keys():
+        assert key in latest_messages_on_shard
+
+    shards_forkchoice = {
+        shard_id: get_shard_fork_choice(
+            starting_blocks[shard_id],
+            children,
+            latest_messages_on_shard[shard_id],
+            shard_id
+        ) for shard_id in starting_blocks
+    }
+
+    return shards_forkchoice

casper/protocols/sharding/forkchoice.py

@@ -0,0 +1,105 @@
+"""The blockchain plot tool implements functions for plotting sharded blockchain data structures"""
+
+from casper.plot_tool import PlotTool
+from casper.safety_oracles.clique_oracle import CliqueOracle
+import casper.utils as utils
+
+
+class ShardingPlotTool(PlotTool):
+    """The module contains functions for plotting a blockchain data structure"""
+
+    def __init__(self, display, save, view, validator_set):
+        super().__init__(display, save, 's')
+        self.view = view
+        self.validator_set = validator_set
+        self.starting_blocks = self.view.starting_blocks
+        self.message_fault_tolerance = dict()
+
+        self.blockchain = []
+        self.communications = []
+
+        self.block_fault_tolerance = {}
+        self.message_labels = {}
+        self.justifications = {
+            validator: []
+            for validator in validator_set
+        }
+
+    def update(self, new_messages=None):
+        """Updates displayable items with new messages and paths"""
+        return
+
+        if new_messages is None:
+            new_messages = []
+
+        self._update_new_justifications(new_messages)
+        self._update_blockchain(new_messages)
+        self._update_block_fault_tolerance()
+        self._update_message_labels(new_messages)
+
+    def plot(self):
+        """Builds relevant edges to display and creates next viewgraph using them"""
+        return
+        best_chain_edge = self.get_best_chain()
+
+        validator_chain_edges = self.get_validator_chains()
+
+        edgelist = []
+        edgelist.append(utils.edge(self.blockchain, 2, 'grey', 'solid'))
+        edgelist.append(utils.edge(self.communications, 1, 'black', 'dotted'))
+        edgelist.append(best_chain_edge)
+        edgelist.extend(validator_chain_edges)
+
+        self.next_viewgraph(
+            self.view,
+            self.validator_set,
+            edges=edgelist,
+            message_colors=self.block_fault_tolerance,
+            message_labels=self.message_labels
+        )
+
+    def get_best_chain(self):
+        """Returns an edge made of the global forkchoice to genesis"""
+        best_message = self.view.estimate()
+        best_chain = utils.build_chain(best_message, None)[:-1]
+        return utils.edge(best_chain, 5, 'red', 'solid')
+
+    def get_validator_chains(self):
+        """Returns a list of edges main from validators current forkchoice to genesis"""
+        vals_chain_edges = []
+        for validator in self.validator_set:
+            chain = utils.build_chain(validator.my_latest_message(), None)[:-1]
+            vals_chain_edges.append(utils.edge(chain, 2, 'blue', 'solid'))
+
+        return vals_chain_edges
+
+    def _update_new_justifications(self, new_messages):
+        for message in new_messages:
+            sender = message.sender
+            for validator in message.justification:
+                last_message = self.view.justified_messages[message.justification[validator]]
+                # only show if new justification
+                if last_message not in self.justifications[sender]:
+                    self.communications.append([last_message, message])
+                    self.justifications[sender].append(last_message)
+
+    def _update_blockchain(self, new_messages):
+        for message in new_messages:
+            if message.estimate is not None:
+                self.blockchain.append([message, message.estimate])
+
+    def _update_message_labels(self, new_messages):
+        for message in new_messages:
+            self.message_labels[message] = message.sequence_number
+
+    def _update_block_fault_tolerance(self):
+        tip = self.view.estimate()
+
+        while tip and self.block_fault_tolerance.get(tip, 0) != len(self.validator_set) - 1:
+            oracle = CliqueOracle(tip, self.view, self.validator_set)
+            fault_tolerance, num_node_ft = oracle.check_estimate_safety()
+
+            if fault_tolerance > 0:
+                self.block_fault_tolerance[tip] = num_node_ft
+
+            tip = tip.estimate

casper/protocols/sharding/sharding_plot_tool.py

@@ -0,0 +1,56 @@
+from casper.protocols.sharding.sharding_view import ShardingView
+from casper.protocols.sharding.block import Block
+from casper.protocols.sharding.sharding_plot_tool import ShardingPlotTool
+from casper.protocol import Protocol
+
+
+class ShardingProtocol(Protocol):
+    View = ShardingView
+    Message = Block
+    PlotTool = ShardingPlotTool
+
+    shard_genesis_blocks = dict()
+    curr_shard_idx = 0
+    curr_shard_ids = ['']
+
+    """Shard ID's look like this:
+           ''
+         /    \
+       '0'    '1'
+      /  \    /  \
+    '00''01''10''11'
+
+
+     Blocks can be merge mined between shards if
+     there is an edge between shards
+     That is, for ids shard_1 and shard_2, there can be a merge block if
+     abs(len(shard_1) - len(shard_2)) = 1 AND
+     for i in range(min(len(shard_1), len(shard_2))):
+        shard_1[i] = shard_2[i]
+    """
+
+    @classmethod
+    def initial_message(cls, validator):
+        """Returns a starting block for a shard"""
+        shard_id = cls.get_next_shard_id()
+
+        estimate = {'prev_blocks': set([None]), 'shard_ids': set([shard_id])}
+        cls.shard_genesis_blocks[shard_id] = Block(estimate, dict(), validator, -1, 0)
+
+        return cls.shard_genesis_blocks['']
+
+    @classmethod
+    def get_next_shard_id(cls):
+        next_id = cls.curr_shard_ids[cls.curr_shard_idx]
+        cls.curr_shard_idx += 1
+
+        if cls.curr_shard_idx == len(cls.curr_shard_ids):
+            next_ids = []
+            for shard_id in cls.curr_shard_ids:
+                next_ids.append(shard_id + '0')
+                next_ids.append(shard_id + '1')
+
+            cls.curr_shard_idx = 0
+            cls.curr_shard_ids = next_ids
+
+        return next_id