Skip to content

Commit

Permalink
Made DiskDGraph reloadable.
Browse files Browse the repository at this point in the history
  • Loading branch information
@pbloem
pbloem committed Aug 24, 2016
1 parent 4e90420 commit bd42174
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 44 deletions.
94 changes: 74 additions & 20 deletions nodes/src/main/java/org/nodes/DiskDGraph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;

import javax.management.RuntimeErrorException;
Expand All @@ -50,6 +51,10 @@
* A version of the LightDGraph which stores all data on disk. Ie. it's slower,
* but can store much bigger graphs.
*
*
* The db file can be stored and re-used later. Make sure to close the graph with
* close().
*
* @author Peter
*
* @param <L>
Expand Down Expand Up @@ -88,25 +93,33 @@ public DiskDGraph(File dir)

/**
*
* @param dir
* @param dbFile The file containing the graph structure. If the file doesn't exist,
* it will be created. IF it does exist, the graph it contains will be loaded.
* @param nullLabels If true, all labels will be null (saving some space).
* Adding a node with a nonnull label will result in an exception.
*/
public DiskDGraph(File dir, boolean nullLabels)
public DiskDGraph(File dbFile, boolean nullLabels)
{
this.nullLabels = nullLabels;

dir.mkdirs();
File dbFile = new File(dir, "graph."+id+".db");

db = DBMaker.fileDB(dbFile).make();

labels = nullLabels ? null : db.indexTreeList("labels", Serializer.STRING).createOrOpen();

in = db.indexTreeList("in", new SerializerIntList()).createOrOpen();
out = db.indexTreeList("out", new SerializerIntList()).createOrOpen();

if(!nullLabels && labels.size() != in.size())
throw new IllegalStateException("labels list has size "+ labels.size() + ", should be " + in.size() + ".");

if(db.exists("numLinks"))
numLinks = db.atomicInteger("numLinks").createOrOpen().get();
else
for(List<Integer> list : in)
numLinks += list.size();
}


@Override
public int size()
{
Expand Down Expand Up @@ -163,7 +176,7 @@ public void remove()

for(int i : series(in.size()))
{
List<Integer> neighbors = in.get(i);
List<Integer> neighbors = new ArrayList<Integer>(in.get(i));

Iterator<Integer> it = neighbors.iterator();
while(it.hasNext())
Expand All @@ -174,7 +187,7 @@ public void remove()
}
for(int i : series(out.size()))
{
List<Integer> neighbors = out.get(i);
List<Integer> neighbors = new ArrayList<Integer>(out.get(i));

Iterator<Integer> it = neighbors.iterator();
while(it.hasNext())
Expand All @@ -188,7 +201,7 @@ public void remove()
// is higher than the one we just removed.
for(int i : series(in.size()))
{
List<Integer> neighbors = in.get(i);
List<Integer> neighbors = new ArrayList<Integer>(in.get(i));

for(int j : series(neighbors.size()))
{
Expand All @@ -201,7 +214,7 @@ public void remove()
}
for(int i : series(out.size()))
{
List<Integer> neighbors = out.get(i);
List<Integer> neighbors = new ArrayList<Integer>(out.get(i));

for(int j : series(neighbors.size()))
{
Expand Down Expand Up @@ -390,21 +403,21 @@ public void disconnect(Node<String> other)

int links = 0;

List<Integer> myOut = out.get(mine);
List<Integer> myOut = new ArrayList<Integer>(out.get(mine));
while(myOut.remove((Integer)his))
links++;
out.set(mine, myOut);

List<Integer> hisOut = out.get(his);
List<Integer> hisOut = new ArrayList<Integer>(out.get(his));
while(hisOut.remove((Integer)mine))
links++;
out.set(his, hisOut);

List<Integer> myIn = in.get(mine);
List<Integer> myIn = new ArrayList<Integer>(in.get(mine));
while(myIn.remove((Integer)his));
in.set(mine, myIn);

List<Integer> hisIn = in.get(his);
List<Integer> hisIn = new ArrayList<Integer>(in.get(his));
while(hisIn.remove((Integer)mine));
in.set(his, hisIn);

Expand Down Expand Up @@ -651,8 +664,14 @@ public Graph<String> graph()
public void remove()
{
check();
in.get(to.index()).remove((Integer)from.index());
out.get(from.index()).remove((Integer)to.index());

List<Integer> list = new ArrayList<Integer>(in.get(to.index()));
list.remove((Integer)from.index());
in.set(to.index(), list);

list = new ArrayList<Integer>(out.get(from.index()));
list.remove((Integer)to.index());
out.set(from.index(), list);

modCount++;
dead = true;
Expand Down Expand Up @@ -1155,6 +1174,28 @@ public List<DNode<String>> neighborsFast(Node<String> node)
return new NodeList(indices);
}

/**
* Loads a previous converted graph.
*
* @param dbFile
* @return
* @throws IOException
*/
public static DiskDGraph fromDB(File dbFile)
throws IOException
{
DB db = DBMaker.fileDB(dbFile).make();

if(db.exists("labels"))
{
db.close();
return new DiskDGraph(dbFile, false);
}

db.close();
return new DiskDGraph(dbFile, true);
}

/**
* Reads a (large) edgelist-encoded file into a DiskDGraph.
*
Expand All @@ -1165,16 +1206,23 @@ public List<DNode<String>> neighborsFast(Node<String> node)
public static DiskDGraph fromFile(File file, File dir)
throws IOException
{
DiskDGraph graph = new DiskDGraph(dir, true);
int id = (new Random()).nextInt(10000000);

return fromFile(file, dir, new File("graph."+id+".db"));
}
public static DiskDGraph fromFile(File file, File tmpDir, File dbFile)
throws IOException
{
DiskDGraph graph = new DiskDGraph(dbFile, true);

// * sort the input file by first element
File forward = new File(dir, "forward.edgelist");
File forward = new File(tmpDir, "forward.edgelist");


List<File> files = ExternalSort.sortInBatch(
file,
new LComp(true), ExternalSort.DEFAULTMAXTEMPFILES,
Charset.defaultCharset(), dir, false);
Charset.defaultCharset(), tmpDir, false);
ExternalSort.mergeSortedFiles(files, forward, new LComp(true), Charset.defaultCharset());

System.out.println("Forward sort finished");
Expand All @@ -1184,12 +1232,12 @@ public static DiskDGraph fromFile(File file, File dir)
System.out.println("Forward list read");

forward.delete();
File backward = new File(dir, "backward.edgelist");
File backward = new File(tmpDir, "backward.edgelist");

files = ExternalSort.sortInBatch(
file,
new LComp(false), ExternalSort.DEFAULTMAXTEMPFILES,
Charset.defaultCharset(), dir, false);
Charset.defaultCharset(), tmpDir, false);
ExternalSort.mergeSortedFiles(files, backward, new LComp(false), Charset.defaultCharset());

System.out.println("Backward sort finished");
Expand All @@ -1216,6 +1264,12 @@ public static DiskDGraph fromFile(File file, File dir)
return graph;
}

public void close()
{
db.atomicInteger("numLinks").createOrOpen().set(numLinks);
db.close();
}

private static long readSorted(List<List<Integer>> list, File file, boolean forward)
throws IOException
{
Expand Down
Loading

0 comments on commit bd42174

Please sign in to comment.