Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Publish milestone together with virtual transactions #188 #200

Open
wants to merge 7 commits into
base: dev-vtx
Choose a base branch
from
Open

Publish milestone together with virtual transactions #188 #200

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c2bd10e
Merge pull request #2 from HelixNetwork/dev
cristina-vasiu Oct 2, 2019
752f17d
Publish milestone together with virtual transactions #188
Oct 3, 2019
39fb050
Fix code review issues (change tree depth, extract constant)
Oct 10, 2019
bc91d28
Reconstruct virtual transactions locally #189
Oct 15, 2019
6290a45
Refactor Merkle
Oct 16, 2019
588dd0f
Removed unused imports
Oct 17, 2019
1368ef6
Add padding NULL leaves for Merkle
Oct 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions src/main/java/net/helix/pendulum/TransactionValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,6 @@ public int getMinWeightMagnitude() {
*/
private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
// ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone
if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) {
return false;
}
if (transactionViewModel.getAttachmentTimestamp() == 0) {
return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash())
|| transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE;
Expand All @@ -155,6 +152,14 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
|| transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS;
}

private boolean isTransactionRequested(TransactionViewModel transactionViewModel) {
if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) {
//todo if is virtual compute it locally
return true;
}
return false;
}

/**
* Runs the following validation checks on a transaction:
* <ol>
Expand All @@ -174,15 +179,17 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) {
public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) {
transactionViewModel.setMetadata();
transactionViewModel.setAttachmentData();

if (!transactionViewModel.isVirtual() && isTransactionRequested(transactionViewModel)) {
log.error("Waiting for transaction... " + transactionViewModel.getHash());
throw new IllegalStateException("Transaction is requested");
}
if(hasInvalidTimestamp(transactionViewModel)) {
throw new StaleTimestampException("Invalid transaction timestamp.");
}
for (int i = VALUE_OFFSET + VALUE_USABLE_SIZE; i < VALUE_OFFSET + VALUE_SIZE; i++) { // todo always false.
if (transactionViewModel.getBytes()[i] != 0) {
throw new IllegalStateException("Invalid transaction value");
}
if(transactionViewModel.isVirtual()){
return;
}

int weightMagnitude = transactionViewModel.weightMagnitude;
if((weightMagnitude < minWeightMagnitude)) {
throw new IllegalStateException("Invalid transaction hash");
Expand All @@ -203,7 +210,8 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m
* @throws RuntimeException if validation fails
*/
public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude) {
TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(SpongeFactory.Mode.S256)));
TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, SpongeFactory.Mode.S256);

runValidation(transactionViewModel, minWeightMagnitude);
return transactionViewModel;
}
Expand Down
36 changes: 36 additions & 0 deletions src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package net.helix.pendulum.controllers;

import net.helix.pendulum.crypto.SpongeFactory;
import net.helix.pendulum.model.*;
import net.helix.pendulum.model.persistables.*;
import net.helix.pendulum.service.milestone.MilestoneTracker;
Expand Down Expand Up @@ -148,6 +149,28 @@ public TransactionViewModel(final byte[] bytes, Hash hash) throws RuntimeExcepti
transaction.type = FILLED_SLOT;
}

/**
* Constructor with transaction bytes and transaction hash.
* @param bytes transaction bytes
* @param mode mode of the hash function used to compute transaction hash
*/
public TransactionViewModel(final byte[] bytes, SpongeFactory.Mode mode) throws RuntimeException {
transaction = new Transaction();
transaction.bytes = new byte[SIZE];
System.arraycopy(bytes, 0, transaction.bytes, 0, SIZE);
if (isVirtual()) {
byte[] buffer = new byte[2 * Hash.SIZE_IN_BYTES];
System.arraycopy(bytes, BRANCH_TRANSACTION_OFFSET, buffer, 0, Hash.SIZE_IN_BYTES);
System.arraycopy(bytes, TRUNK_TRANSACTION_OFFSET, buffer, Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES);
this.hash = TransactionHash.calculate(buffer, 0, buffer.length, SpongeFactory.create(mode));
} else {
this.hash = TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(mode));
}

weightMagnitude = this.hash.leadingZeros();
transaction.type = FILLED_SLOT;
}

/**
* Get the number of transactins in the database.
* @param tangle
Expand Down Expand Up @@ -675,6 +698,19 @@ public boolean isMilestone() {
return transaction.milestone;
}

/**
* The {@link Transaction#milestone} flag indicates if the {@link Transaction} is a virtual transaction
* @return true if the {@link Transaction} is virtual and false otherwise
*/
public boolean isVirtual() {
byte[] bundleNonceForVirtual = new byte[Hash.SIZE_IN_BYTES];
Arrays.fill(bundleNonceForVirtual, (byte) 0xff);
if (getBundleNonceHash().equals(HashFactory.BUNDLENONCE.create(bundleNonceForVirtual)) && Arrays.equals(getNonce(), new byte[NONCE_SIZE])) {
return true;
}
return false;
}

public TransactionViewModel isMilestoneBundle(Tangle tangle) throws Exception{
for (Hash bundleTx : BundleViewModel.load(tangle, getBundleHash()).getHashes()){
TransactionViewModel tx = TransactionViewModel.fromHash(tangle, bundleTx);
Expand Down
67 changes: 58 additions & 9 deletions src/main/java/net/helix/pendulum/crypto/Merkle.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
import net.helix.pendulum.controllers.TransactionViewModel;
import net.helix.pendulum.model.Hash;
import net.helix.pendulum.model.HashFactory;
import net.helix.pendulum.utils.bundle.BundleUtils;
import org.bouncycastle.util.encoders.Hex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.ByteBuffer;
Expand All @@ -14,6 +17,8 @@

public class Merkle {

private static final Logger log = LoggerFactory.getLogger(Merkle.class);

public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) {
int index = indexIn;
final Sponge sha3 = SpongeFactory.create(mode);
Expand Down Expand Up @@ -55,6 +60,44 @@ public static List<List<Hash>> buildMerkleKeyTree(String seed, int pubkeyDepth,
}


public static List<byte[]> buildMerkleTransactionTree(List<Hash> leaves, Hash milestoneHash, Hash address){
if (leaves.isEmpty()) {
leaves.add(Hash.NULL_HASH);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There were some concerns from Oliver about using NULL_HASH. Is it kosher here or we should use another filler?

Copy link
Contributor Author

@cristina-vasiu cristina-vasiu Oct 10, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding NULL_HASH it can be replaced with other hash (but should exists in tangle), here it means that no tips are found, so their merkle root is NULL_HASH, in tangle it will look like this milestone is directly connected to genesis.

Copy link
Contributor

@oracle58 oracle58 Oct 22, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in tangle it will look like this milestone is directly connected to genesis

why I don't think it is kosher.

Copy link
Contributor

@oracle58 oracle58 Oct 22, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can be replaced with other hash (but should exists in tangle)

yes it has to be a valid tx_hash, not sure how, without implementing a(nother) genesis tx.

}
byte[] buffer;
Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256);
int depth = (int) Math.ceil(Math.sqrt(leaves.size()));
int row = 1;
List<byte[]> virtualTransactionList = new ArrayList<byte[]>();
while (leaves.size() > 1) {
List<Hash> nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]);
for (int i = 0; i < nextKeys.size(); i++) {
if (areLeavesNull(leaves, i)) continue;
sha3.reset();
Hash k1 = leaves.get(i * 2);
Hash k2 = leaves.get(i * 2 + 1);
buffer = computeParentHash(sha3, k1, k2);
Hash parentHash = HashFactory.TRANSACTION.create(buffer);
nextKeys.set(i, parentHash);
// if( i == 0) add only one virtual transaction for each level
virtualTransactionList.add(createVirtualTransaction(k1, k2, row, milestoneHash, address, parentHash));
}
leaves = nextKeys;
row++;
}
return virtualTransactionList;
}

private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) {
byte[] buffer;
buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32);
sha3.absorb(buffer, 0, buffer.length);
buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32);
cristina-vasiu marked this conversation as resolved.
Show resolved Hide resolved
sha3.absorb(buffer, 0, buffer.length);
sha3.squeeze(buffer, 0, buffer.length);
return buffer;
}

public static List<List<Hash>> buildMerkleTree(List<Hash> leaves){
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What are the pros and cons of having List<List<Hash>> for the tree representation against a flat array (ArrayList<Hash>) ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It depends on what we want to use it:
List<List> is faster if you need to access hashes for a certain level of Merkle tree, but it's slower on traverse all nodes (without any order), or traverse from leves to root.
List is faster if all nodes should be traversed and if merkle level is not relevant
In most cases I saw that only root is used, so maybe we can change the calling methods to call getMerkleRoot instead of the entire tree.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The trick is to use the compact array representation of a binary tree. If you have say 8 leaves, you pack it into array of size 15 = (2*8) - 1 in the following way:

  • leaves have indices 7, 8, ..., 14
  • next level indices 3,4,5,6
  • next level 1,2
  • root 0

The level i has indices from 2^i-1 to 2^{i+1}-2
The benefit of having this representation is that you initially allocate the full size and fill evertyhing with null and as the data becomes available, you can quickly navigate to parents and children by simple bit shifts of the indices, at the same time preserving the order at all levels.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To be more precise: children of a[i] are a[2*i+1] and a[2*i+2], parent of a[i] is a[(i-1) >> 1]

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd suggest introducing a separate interface MerkleTree and MerkleTreeImpl, where this builder can be moved.

Also, I don't think that we should replace null with NULL_HASH at all levels, it should be used only for padding the leaves list so that it's size is an integer of the form 2^i for some i.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no padding to merkle tree leaves, it just take the leaves and pair them 2 by 2 and compute the hashes, when there is an odd number of leaves the last one is not taken into consideration.

pendulum/src/main/java/net/helix/pendulum/crypto/Merkle.java

Line 71 in a148f48

List<Hash> nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]);

if (leaves.isEmpty()) {
leaves.add(Hash.NULL_HASH);
Expand All @@ -70,18 +113,11 @@ public static List<List<Hash>> buildMerkleTree(List<Hash> leaves){
// Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them
cristina-vasiu marked this conversation as resolved.
Show resolved Hide resolved
List<Hash> nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]);
for (int i = 0; i < nextKeys.size(); i++) {
if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) {
// leave the combined key null as well
continue;
}
if (areLeavesNull(leaves, i)) continue;
sha3.reset();
Hash k1 = leaves.get(i * 2);
Hash k2 = leaves.get(i * 2 + 1);
buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32);
sha3.absorb(buffer, 0, buffer.length);
buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32);
sha3.absorb(buffer, 0, buffer.length);
sha3.squeeze(buffer, 0, buffer.length);
buffer = computeParentHash(sha3, k1, k2);
nextKeys.set(i, HashFactory.TRANSACTION.create(buffer));
}
leaves = nextKeys;
Expand All @@ -90,6 +126,19 @@ public static List<List<Hash>> buildMerkleTree(List<Hash> leaves){
return merkleTree;
}

private static boolean areLeavesNull(List<Hash> leaves, int i) {
if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) {
// leave the combined key null as well
return true;
}
return false;
}

private static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address, Hash transactionHash) {
log.debug("New virtual transaction + " + transactionHash + " for milestone: " + milestoneHash + " with merkle index: " + merkleIndex + " [" + branchHash + ", " + trunkHash + " ]");
return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash, address);
}

public static boolean validateMerkleSignature(List<TransactionViewModel> bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) {

//System.out.println("Validate Merkle Signature");
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/net/helix/pendulum/network/TransactionRequester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ public void requestTransaction(Hash hash, boolean milestone) throws Exception {
protected void popEldestTransactionToRequest() {
Iterator<Hash> iterator = transactionsToRequest.iterator();
if (iterator.hasNext()) {
iterator.next();
Hash transactionHash = iterator.next();
log.debug("Transaction: " + transactionHash + " has been removed from requestList!");
iterator.remove();
}
}
Expand Down
Loading