diff --git a/app/pom.xml b/app/pom.xml index 429393a..3f88fc8 100644 --- a/app/pom.xml +++ b/app/pom.xml @@ -5,7 +5,7 @@ pad-fs com.dido.code.pad - 1.0-SNAPSHOT + 1.0 4.0.0 @@ -15,7 +15,7 @@ com.dido.code.pad core - 1.0-SNAPSHOT + 1.0 junit diff --git a/app/src/main/java/com/dido/pad/app/AppRunner.java b/app/src/main/java/com/dido/pad/app/AppRunner.java index 94ede2f..98e9545 100644 --- a/app/src/main/java/com/dido/pad/app/AppRunner.java +++ b/app/src/main/java/com/dido/pad/app/AppRunner.java @@ -75,14 +75,14 @@ public static void main(String[] args) throws IOException, InterruptedException case ("disconnect"): String nodeDisconnect= cmds[1]; if(clients.containsKey(nodeDisconnect)){ - clients.get(nodeDisconnect).disconnect(); + (clients.get(nodeDisconnect)).disconnect(); } else System.out.println("node is not present"); break; case ("connect"): String nodeConnect= cmds[1]; if(clients.containsKey(nodeConnect)){ - clients.get(nodeConnect).connect(); + (clients.get(nodeConnect)).connect(); }else System.out.println("node is not present"); diff --git a/cli/pom.xml b/cli/pom.xml index 97a221e..b577328 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -5,7 +5,7 @@ pad-fs com.dido.code.pad - 1.0-SNAPSHOT + 1.0 4.0.0 @@ -14,7 +14,7 @@ com.dido.code.pad core - 1.0-SNAPSHOT + 1.0 diff --git a/cli/src/main/java/com/dido/pad/cli/client/Client.java b/cli/src/main/java/com/dido/pad/cli/client/Client.java index d09f174..30b8e18 100644 --- a/cli/src/main/java/com/dido/pad/cli/client/Client.java +++ b/cli/src/main/java/com/dido/pad/cli/client/Client.java @@ -81,7 +81,8 @@ private void gossipClient() { Node node = clientService.getRandomNode(); - byte buf[] = Helper.fromClientMsgtoByte(reqNodes); + //byte buf[] = Helper.fromClientMsgtoByte(reqNodes); + byte buf[] = Helper.fromAppMsgtoByte(reqNodes); InetAddress destAddress = InetAddress.getByName(node.getIpAddress()); DatagramPacket packet = new DatagramPacket(buf, buf.length, destAddress, Helper.STORAGE_PORT); clientSocket.send(packet); diff --git a/copyJarCnr.pl b/copyJarCnr.pl index dd28aaf..37e1e30 100644 --- a/copyJarCnr.pl +++ b/copyJarCnr.pl @@ -1,25 +1,46 @@ use strict; -use Net::SCP qw(scp iscp); -use Net::SSH qw(ssh); -use Log::Dispatch::Syslog; +use Net::SCP; + +# novello 146.48.82.73 -# novello 146.48.82.73 # node1 146.48.82.91 node1.novello.isti.cnr.it # node2 146.48.82.84 # node3 146.48.82.75 # node4 146.48.82.76 -# node5 146.48.82.93 +# node5 146.48.82.93 (client) + +my %servers = ( + "node1" => "146.48.82.91", + "node2" => "146.48.82.84", + "node3" => "146.48.82.75", + "node4" => "146.48.82.76", + "node5" => "146.48.82.93" +); + +foreach my $key (keys %servers){ + my $scp = Net::SCP->new( "$key.novello.isti.cnr.it", "aspirantidottori" ); + # $scp->cwd(); + `echo 'java -cp core-1.0-jar-with-dependencies.jar com.dido.pad.PadFsNode -ip $servers{$key} -id $key 146.48.82.91:node1 146.48.82.84:node2' > start.sh`; + `chmod +x start.sh`; + + `echo 'rm /tmp/node*' > eraseData.sh`; + `chmod +x eraseData.sh`; + + $scp->put('eraseData.sh') or die $scp->{errstr}; + + $scp->put('start.sh') or die $scp->{errstr}; + $scp->put("core/target/core-1.0-jar-with-dependencies.jar") or die $scp->{errstr}; + print ("Loaded PadFsNode into $key \n"); +} -#declare local variables -my $scp; -my $host = "node1.novello.isti.cnr.it"; -my $user = "aspirantidottori"; -my $remotedir = "/home/"; -my $file = "test.txt"; -my $cmd = "/bin/ls"; +my $scp = Net::SCP->new( "node5.novello.isti.cnr.it", "aspirantidottori" ); +#$scp->cwd('luca'); +`echo 'java -cp cli-1.0-jar-with-dependencies.jar com.dido.pad.cli.MainClient -ip 146.48.82.93 -id client 146.48.82.91:node1' > start-client.sh`; +`chmod +x start-client.sh`; + $scp->put('start-client.sh') or die $scp->{errstr}; +$scp->put("cli/target/cli-1.0-jar-with-dependencies.jar") or die $scp->{errstr}; +print ("Loaded Client into node5 \n"); -######first connect to $host via Net::SSH and run /bin/ls########### -ssh("$user\@$host",$cmd); +`rm start.sh && rm start-client.sh && rm eraseData.sh`` -######first connect to $host via Net::SSH and copy file $file########### diff --git a/core/pom.xml b/core/pom.xml index de9b86b..1984e30 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -5,7 +5,7 @@ pad-fs com.dido.code.pad - 1.0-SNAPSHOT + 1.0 4.0.0 diff --git a/core/src/main/java/com/dido/pad/Node.java b/core/src/main/java/com/dido/pad/Node.java index da2d3ee..cf43076 100644 --- a/core/src/main/java/com/dido/pad/Node.java +++ b/core/src/main/java/com/dido/pad/Node.java @@ -28,6 +28,7 @@ public class Node { private int portStorage; private int portGossip; + private int numReplicas; @@ -92,8 +93,7 @@ public void connect() { _storageService.connect(); } - public void disconnect() { - _storageService.disconnect(); + public void disconnect() { _storageService.disconnect(); } public GossipManager getGossipmanager() { diff --git a/core/src/main/java/com/dido/pad/PadFsNode.java b/core/src/main/java/com/dido/pad/PadFsNode.java index 199a9fd..b153637 100644 --- a/core/src/main/java/com/dido/pad/PadFsNode.java +++ b/core/src/main/java/com/dido/pad/PadFsNode.java @@ -28,15 +28,16 @@ public static void main(String[] args){ Node node = new Node(jct.getIp(), jct.getId(), jct.getStoragePort(), jct.getGossipPort(), jct.getGossipMember(), new GossipSettings()); node.start(); + System.out.println("\nInsert a command [h for usage message]...\n"); BufferedReader bufferReader = new BufferedReader(new InputStreamReader(System.in)); - String help = "ID "+ node.getId() + " usage: \n " + + String help = "\n["+ node.getId() + "] usage: \n" + "\tdisconnect : disconnect the node from the network (it will not respond to message)\n" + - "\tconnect : connect the node into the network "; + "\tconnect : connect the node into the network \n"+ + "\th : show this help"; while(true) { - System.out.println("\n Insert a command [h for usage message]..."); String input = null; try { input = bufferReader.readLine(); diff --git a/core/src/main/java/com/dido/pad/StorageService.java b/core/src/main/java/com/dido/pad/StorageService.java index 508ae88..fc50a7b 100644 --- a/core/src/main/java/com/dido/pad/StorageService.java +++ b/core/src/main/java/com/dido/pad/StorageService.java @@ -355,7 +355,7 @@ private void manageAppRequest(RequestAppMsg msg) { Versioned bkuData = msgReply.getData(); switch (bkuData.compareTo(myData)) { case BEFORE: //my data version is newer than the backup version - LOGGER.info(myNode.getIpAddress() + " - my version is BEFORE with <" + msgReply.getData().getData().getKey() + " : " + msgReply.getData().getVersion() + " > from " + msgReply.getIpSender()); + LOGGER.info(myNode.getIpAddress() + " - My version is BEFORE with <" + msgReply.getData().getData().getKey() + " : " + msgReply.getData().getVersion() + " > from " + msgReply.getIpSender()); send(msg.getIpSender(), Helper.STORAGE_PORT, new ReplyAppMsg(Msg.OP.OK, " GET " + myData.getData().toString())); //for (Node backup : preferenceNodes) { for (Node backup : cHasher.getNextServers(myNode, SIZE_PREF_LIST)) { @@ -363,12 +363,12 @@ private void manageAppRequest(RequestAppMsg msg) { } break; case AFTER: //my data version is older than the backup version - LOGGER.info(myNode.getIpAddress() + " - my version is AFTER with<" + msgReply.getData().getData().getKey() + "> version: " + msgReply.getData().getVersion() + " from " + msgReply.getIpSender()); + LOGGER.info(myNode.getIpAddress() + " - My version is AFTER with<" + msgReply.getData().getData().getKey() + "> version: " + msgReply.getData().getVersion() + " from " + msgReply.getIpSender()); myData.mergeTo(bkuData); send(msg.getIpSender(), Helper.STORAGE_PORT, new ReplyAppMsg(Msg.OP.OK, " GET " + bkuData.getData().toString())); break; case CONCURRENT: //concurrent version must be resolved by the client - LOGGER.info(myNode.getIpAddress() + " - my version is CONCURRENT with <" + msgReply.getData().getData().getKey() + "> version: " + msgReply.getData().getVersion() + " from " + msgReply.getIpSender()); + LOGGER.info(myNode.getIpAddress() + " - My version is CONCURRENT with <" + msgReply.getData().getData().getKey() + "> version: " + msgReply.getData().getVersion() + " from " + msgReply.getIpSender()); String selection = " 1 : " + myData.getData().toString() + " \n 2 : " + bkuData.getData().toString(); Msg conflict = new RequestConflictMsg(Msg.TYPE.REQUEST, Msg.OP.GET, myNode.getIpAddress(), Helper.STORAGE_PORT, selection); sendConflict(msg.getIpSender(), conflict, Helper.CONFLICT_LISTEN_PORT); diff --git a/core/src/test/java/com/dido/pad/TestQuorum.java b/core/src/test/java/com/dido/pad/TestQuorum.java index ba31cbb..f48f113 100644 --- a/core/src/test/java/com/dido/pad/TestQuorum.java +++ b/core/src/test/java/com/dido/pad/TestQuorum.java @@ -24,6 +24,7 @@ public class TestQuorum { int clusterMembers = 3; + public TestQuorum(){ startupMembers.add(new RemoteGossipMember("127.0.0.1", Helper.GOSSIP_PORT, "node1")); for (int i = 1; i < clusterMembers + 1; ++i) { @@ -40,7 +41,7 @@ public void setUP(){ } - //@Test + @Test public void testGetMerge() { try { diff --git a/docs/Pad-fs_report/master.pdf b/docs/Pad-fs_report/master.pdf new file mode 100644 index 0000000..8c291e1 Binary files /dev/null and b/docs/Pad-fs_report/master.pdf differ diff --git a/docs/Pad-fs_report/sections/design.tex b/docs/Pad-fs_report/sections/design.tex index 33fd2e1..250507a 100644 --- a/docs/Pad-fs_report/sections/design.tex +++ b/docs/Pad-fs_report/sections/design.tex @@ -5,9 +5,9 @@ \begin{itemize} \item \textbf{Partitioning}. In order to scale incrementally pad-fs use \textit{consistent hashing} . The hash function can be defined by the programmer, by default \emph{Pad-fs} uses \textit{SHA1} hash function for hashing both the data and the nodes. -\item \textbf{Replication}. In \emph{Pad-fs} a data is replicated in multiple distinct nodes: into the master node and into \texttt{N\_REPLICAS} backup nodes in the clockwise direction. The master node is responsable to manage the keys. \texttt{N\_REPLICAS} is the variable that indicates the number of nodes after the master node that has to receive a copy of the data (by default \texttt{N\_REPLICAS}=2). +\item \textbf{Replication}. In \emph{Pad-fs} a data is replicated in \texttt{N\_REPLICAS} distinct nodes: the master node and backups nodes in the clockwise direction. The master node is responsable to manage the keys. \texttt{N\_REPLICAS} is the variable that indicates the number of nodes that receive a copy of the data (by default \texttt{N\_REPLICAS}=3). -\item \textbf{Quorum system}. It is used for handling temporary failures. \texttt{WRITE\_NODES} are the number of backups nodes that must responds successfully at at put operation. \texttt{READ\_NODES} are the number of backups nodes that must terminates successfully to a get operation. If not all the backups responds a error message is shown to the user and the operation is not performed (by default \texttt{WRITE\_NODES}=1, and \texttt{READ\_NODES} = 2) +\item \textbf{Quorum system}. It is used for handling temporary failures. \texttt{WRITE\_NODES} are the number of backups nodes that must responds successfully at at put operation. \texttt{READ\_NODES} are the number of backups nodes that must terminates successfully to a get operation. If not all the backups responds a error message is shown to the user and the operation is not performed (by default \texttt{WRITE\_NODES}=2, and \texttt{READ\_NODES} = 2) \item \textbf{Versioning}. Pad-fs uses vector clocks associated with the data in order to resolve inconsistency. The vector clock is a pair where id is the node id and \textit{n} is an integer number. diff --git a/docs/Pad-fs_report/sections/implementation.tex b/docs/Pad-fs_report/sections/implementation.tex index adf4945..849b55a 100644 --- a/docs/Pad-fs_report/sections/implementation.tex +++ b/docs/Pad-fs_report/sections/implementation.tex @@ -10,6 +10,11 @@ \section{Execution of get, put, list operations} \item \emph{rm(key)}: removes the key and the associated value in all the node where key is stored. \end{itemize} +\subsection{Aborting operations} +Tha \emph{Pad-fs} use a quorum system. If the quorum system is not respected,the operation are aborted. +As example during a \emph{put} operation: +\begin{•} + \subsubsection*{Get operation} The get(k) retrieve the value associated with the key k. The figure~\ref{fig:get} shows the steps of a successfully get operation. diff --git a/docs/Pad-fs_slides/Pad-fs_Neri.odp b/docs/Pad-fs_slides/Pad-fs_Neri.odp index 4d9b89e..723913d 100644 Binary files a/docs/Pad-fs_slides/Pad-fs_Neri.odp and b/docs/Pad-fs_slides/Pad-fs_Neri.odp differ diff --git a/eraseData.sh b/eraseData.sh new file mode 100755 index 0000000..b7d130d --- /dev/null +++ b/eraseData.sh @@ -0,0 +1 @@ +rm /tmp/node* diff --git a/start-local-system.sh b/start-local-system.sh new file mode 100755 index 0000000..ecba1b6 --- /dev/null +++ b/start-local-system.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +mvn clean -DskipTests=true package + +gnome-terminal -e "java -cp cli/target/cli-1.0-jar-with-dependencies.jar com.dido.pad.cli.MainClient -ip 127.0.0.254 -id client 127.0.0.1:node1" +echo "started Client 127.0.0.254" + +gnome-terminal -e "java -cp core/target/core-1.0-jar-with-dependencies.jar com.dido.pad.PadFsNode -ip 127.0.0.1 -id node1 127.0.0.1:node1" +echo "started node 127.0.0.1" +gnome-terminal -e "java -cp core/target/core-1.0-jar-with-dependencies.jar com.dido.pad.PadFsNode -ip 127.0.0.2 -id node2 127.0.0.1:node1" +echo "started node 127.0.0.2" +gnome-terminal -e "java -cp core/target/core-1.0-jar-with-dependencies.jar com.dido.pad.PadFsNode -ip 127.0.0.3 -id node3 127.0.0.1:node1" +echo "started node 127.0.0.3" +gnome-terminal -e "java -cp core/target/core-1.0-jar-with-dependencies.jar com.dido.pad.PadFsNode -ip 127.0.0.4 -id node4 127.0.0.1:node1" +echo "started node 127.0.0.4" +