Skip to content

Commit

Permalink
docker + while() in storage service
Browse files Browse the repository at this point in the history
  • Loading branch information
@dido18
dido18 committed May 11, 2016
1 parent 0fa16ad commit a3c5697
Show file tree
Hide file tree
Showing 18 changed files with 192 additions and 127 deletions.
21 changes: 0 additions & 21 deletions .idea/PAD-FileSystem.iml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 0 additions & 21 deletions app/app.iml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,6 @@
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: com.github.edwardcapriolo:gossip:master" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson:jackson-datatype-json-org:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.json:json:20090211" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.15" level="project" />
<orderEntry type="library" name="Maven: javax.mail:mail:1.4" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
<orderEntry type="library" name="Maven: org.jdom:jdom:2.0.2" level="project" />
<orderEntry type="library" name="Maven: com.google.collections:google-collections:1.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-joda:2.7.2" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.4" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.beust:jcommander:1.48" level="project" />
<orderEntry type="library" name="Maven: org.mapdb:mapdb:1.0.8" level="project" />
<orderEntry type="module" module-name="core" />
<orderEntry type="library" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
Expand Down
1 change: 1 addition & 0 deletions app/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
<groupId>com.dido.code.pad</groupId>
<version>1.0-SNAPSHOT</version>
</parent>

<modelVersion>4.0.0</modelVersion>

<artifactId>app</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions app/src/main/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM java:alpine
VOLUME /tmp
ADD app.jar app.jar
ENTRYPOINT ["java","-cp","app.jar"]
Binary file added app/src/main/docker/app.jar
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion app/src/main/java/com/dido/pad/app/AppRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public class AppRunner {
public static void main(String[] args) throws IOException, InterruptedException {
//startup gossip member
GossipSettings settings = new GossipSettings();
int seedNodes = 2;
int seedNodes = 1;
List<GossipMember> startupMembers = new ArrayList<>();
for (int i = 1; i < seedNodes + 1; ++i) {
startupMembers.add(new RemoteGossipMember("127.0.0." + i, Helper.GOSSIP_PORT, "node" + i));
Expand Down
21 changes: 0 additions & 21 deletions cli/cli.iml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,6 @@
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: com.github.edwardcapriolo:gossip:master" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson:jackson-datatype-json-org:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.json:json:20090211" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.15" level="project" />
<orderEntry type="library" name="Maven: javax.mail:mail:1.4" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
<orderEntry type="library" name="Maven: org.jdom:jdom:2.0.2" level="project" />
<orderEntry type="library" name="Maven: com.google.collections:google-collections:1.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-joda:2.7.2" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.4" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.beust:jcommander:1.48" level="project" />
<orderEntry type="library" name="Maven: org.mapdb:mapdb:1.0.8" level="project" />
<orderEntry type="module" module-name="core" />
<orderEntry type="library" name="Maven: com.github.edwardcapriolo:gossip:master" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson:jackson-datatype-json-org:1.8.0" level="project" />
Expand Down
4 changes: 4 additions & 0 deletions cli/src/main/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM java:alpine
VOLUME /tmp
ADD cli.jar cli.jar
ENTRYPOINT ["java","-cp","cli.jar"]
Binary file added cli/src/main/docker/cli.jar
View file
Binary file not shown.
2 changes: 1 addition & 1 deletion cli/src/main/java/com/dido/pad/cli/client/Cli.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ public void runCli() {

private void sendRmAndWait(String rmkey) {
Node n = getcHasher().getServerForData(rmkey);
RequestAppMsg msgRm = new RequestAppMsg(Msg.OP.RM,rmkey,"");
RequestAppMsg msgRm = new RequestAppMsg<>(Msg.OP.RM,rmkey,"");
msgRm.setIpSender(client.getIpAddress());
String ip = n.getIpAddress();

Expand Down
21 changes: 0 additions & 21 deletions core/core.iml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,5 @@
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.beust:jcommander:1.48" level="project" />
<orderEntry type="library" name="Maven: org.mapdb:mapdb:1.0.8" level="project" />
<orderEntry type="library" name="Maven: com.github.edwardcapriolo:gossip:master" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson:jackson-datatype-json-org:1.8.0" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.jackson:jackson-core-asl:1.8.6" level="project" />
<orderEntry type="library" name="Maven: org.json:json:20090211" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.15" level="project" />
<orderEntry type="library" name="Maven: javax.mail:mail:1.4" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
<orderEntry type="library" name="Maven: org.jdom:jdom:2.0.2" level="project" />
<orderEntry type="library" name="Maven: com.google.collections:google-collections:1.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-joda:2.7.2" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.4" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.7.2" level="project" />
<orderEntry type="library" name="Maven: com.beust:jcommander:1.48" level="project" />
<orderEntry type="library" name="Maven: org.mapdb:mapdb:1.0.8" level="project" />
</component>
</module>
2 changes: 1 addition & 1 deletion core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
<groupId>com.dido.code.pad</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<modelVersion>4.0.0</modelVersion>
<artifactId>core</artifactId>


Expand Down
4 changes: 4 additions & 0 deletions core/src/main/docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM java:alpine
VOLUME /tmp
ADD core.jar core.jar
ENTRYPOINT ["java","-cp","core.jar"]
Binary file added core/src/main/docker/core.jar
View file
Binary file not shown.
21 changes: 6 additions & 15 deletions core/src/main/java/com/dido/pad/StorageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,11 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.google.code.gossip.GossipMember;
import com.sun.org.apache.regexp.internal.RE;
import org.apache.log4j.Logger;


import java.io.IOException;
import java.net.*;
import java.time.temporal.ValueRange;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
Expand Down Expand Up @@ -74,11 +72,6 @@ public StorageService(Node node, List<GossipMember> seedNodes) {
}

}
/*
public List<Node> getReplicasNodes(Node server, int replicas) {
return cHasher.getNextServers(server, replicas);
}
*/

public PersistentStorage getStorage() {
return storage;
Expand Down Expand Up @@ -109,9 +102,8 @@ public void removeServer(Node n) {
@Override
public void run() {
// problem if the system is up: takes as next the first nodes the goes up.
int g = cHasher.getAllNodes().size();
while (N_REPLICAS < cHasher.getAllNodes().size() && cHasher.getAllNodes().size() < Helper.NETWORK_SIZE) {
LOGGER.info(myNode.getIpAddress() + " - Required ("+ (N_REPLICAS+1) +" < #nodes <=" + Helper.NETWORK_SIZE+ ") nodes, found " + cHasher.getAllNodes().size()+" nodes.");
while (cHasher.getAllNodes().size() < Helper.NETWORK_SIZE ){// N_REPLICAS > cHasher.getAllNodes().size()) {
StorageService.LOGGER.info(myNode.getIpAddress() + " - Required ("+ (N_REPLICAS+1) +" < #nodes <=" + Helper.NETWORK_SIZE+ ") nodes, found " + cHasher.getAllNodes().size()+" nodes.");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Expand Down Expand Up @@ -259,7 +251,7 @@ private void manageSystemRequest(RequestSystemMsg msg) {
else{
String info = " - Impossible to remove <"+keyRm+" >from local database";
LOGGER.info(myNode.getIpAddress() + info);
send(msg.getIpSender(), Helper.QUORUM_PORT, new ReplySystemMsg(Msg.OP.OK, myNode.getIpAddress(), Helper.QUORUM_PORT, info));
send(msg.getIpSender(), Helper.QUORUM_PORT, new ReplySystemMsg(Msg.OP.ERR, myNode.getIpAddress(), Helper.QUORUM_PORT, info));
}
}
}
Expand All @@ -282,7 +274,7 @@ private void manageAppRequest(RequestAppMsg<?> msg) {
newData.getVersion().increment(myNode.getId());

List<ReplySystemMsg> replies = askQuorum(newData, Helper.QUORUM_PORT, Msg.OP.PUT);
if(replies.size() < WRITE_NODES ){//|| checkAllOK(replies)){ // UNDO update operation: send old data in a PUT operation
if(replies.size() < WRITE_NODES || !checkAllOK(replies)){ // UNDO update operation: send old data in a PUT operation
for(Node n : preferenceNodes){
//for(ReplySystemMsg rep : replies){
send(n.getIpAddress(), Helper.STORAGE_PORT, new RequestSystemMsg(Msg.OP.PUT,myNode.getIpAddress(),Helper.STORAGE_PORT, oldData));
Expand Down Expand Up @@ -412,7 +404,7 @@ private void manageAppRequest(RequestAppMsg<?> msg) {
String keyRm = msg.getKey();
if(storage.containsKey(keyRm)) {
List<ReplySystemMsg> replies = askQuorum(storage.get(keyRm),Helper.QUORUM_PORT, Msg.OP.RM);
if(replies.size() < WRITE_NODES ){//|| checkAllOK(replies)){ // not all the writes nodes has responded OR some responds ERR
if(replies.size() < WRITE_NODES || !checkAllOK(replies)){ // not all the writes nodes has responded OR some responds ERR
for(Node n : preferenceNodes){
//for(ReplySystemMsg rep: replies){ //new RequestSystemMsg(Msg.OP.PUT,myNode.getIpAddress(),Helper.STORAGE_PORT, oldData)
send(n.getIpAddress(), Helper.STORAGE_PORT, new RequestSystemMsg(Msg.OP.PUT,myNode.getIpAddress(),Helper.STORAGE_PORT,storage.get(keyRm)));
Expand Down Expand Up @@ -449,7 +441,7 @@ private void manageAppRequest(RequestAppMsg<?> msg) {

private boolean checkAllOK(List<ReplySystemMsg> replies) {
for(ReplySystemMsg msg : replies){
if(msg.getOperation().equals(Msg.OP.ERR))
if(msg.getOperation() == Msg.OP.ERR)
return false;
}
return true;
Expand Down Expand Up @@ -497,7 +489,6 @@ private void send(String destIp, int destPort, Msg msg) {
public List<ReplySystemMsg> askQuorum(Versioned vData, int listenPort, Msg.OP op) {
RequestSystemMsg reqQuorum;


switch (op){
case PUT:
reqQuorum = new RequestSystemMsg(op, myNode.getIpAddress(), 0, vData);
Expand Down
8 changes: 4 additions & 4 deletions core/src/main/java/com/dido/pad/hashing/Hasher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ synchronized public void removeServer(T server) {
}


private ByteBuffer convertAndApplyHash(T server) {
synchronized private ByteBuffer convertAndApplyHash(T server) {
byte[] bucketNameInBytes = hashFunction.hash(nodeToByteConverter.convert(server));
return ByteBuffer.wrap(hashFunction.hash(bucketNameInBytes));
}
Expand Down Expand Up @@ -83,7 +83,7 @@ synchronized public ArrayList<T> getAllNodes() {
return new ArrayList<>(serversMap.values());
}

public boolean containsNode(T node) {
synchronized public boolean containsNode(T node) {
ByteBuffer bbServerVirtuals = convertAndApplyHash(node);
return this.serversMap.containsKey(bbServerVirtuals);
}
Expand All @@ -104,7 +104,7 @@ synchronized public NavigableMap<ByteBuffer, T> getServersMap() {
* @param server starting node server
* @return list of Nodes
*/
public ArrayList<T> getNextServers(T server, int number) {
synchronized public ArrayList<T> getNextServers(T server, int number) {
Preconditions.checkArgument(number > 0 , "number of next servers cannot be negative");
ByteBuffer myByte = convertAndApplyHash(server);
ByteBuffer bbNext= convertAndApplyHash(server);
Expand All @@ -131,7 +131,7 @@ else if (!nexts.contains(serversMap.get(bbNext))&& !bbNext.equals(myByte)) {
}


public ArrayList<T> getPreviousServer(T server, int number){
synchronized public ArrayList<T> getPreviousServer(T server, int number){
// ArrayList<ByteBuffer> virtuals = virtualForServer.get(server);

ByteBuffer bbPrevious = convertAndApplyHash(server); // first entry is the Bytebuffer of the physical server.
Expand Down
Loading

0 comments on commit a3c5697

Please sign in to comment.