Skip to content

Commit bd42174

Browse files
committed
Made DiskDGraph reloadable.
1 parent 4e90420 commit bd42174

File tree

2 files changed

+136
-44
lines changed

2 files changed

+136
-44
lines changed

nodes/src/main/java/org/nodes/DiskDGraph.java

Lines changed: 74 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.LinkedList;
2525
import java.util.List;
2626
import java.util.NoSuchElementException;
27+
import java.util.Random;
2728
import java.util.Set;
2829

2930
import javax.management.RuntimeErrorException;
@@ -50,6 +51,10 @@
5051
* A version of the LightDGraph which stores all data on disk. Ie. it's slower,
5152
* but can store much bigger graphs.
5253
*
54+
*
55+
* The db file can be stored and re-used later. Make sure to close the graph with
56+
* close().
57+
*
5358
* @author Peter
5459
*
5560
* @param <L>
@@ -88,25 +93,33 @@ public DiskDGraph(File dir)
8893

8994
/**
9095
*
91-
* @param dir
96+
* @param dbFile The file containing the graph structure. If the file doesn't exist,
97+
* it will be created. IF it does exist, the graph it contains will be loaded.
9298
* @param nullLabels If true, all labels will be null (saving some space).
9399
* Adding a node with a nonnull label will result in an exception.
94100
*/
95-
public DiskDGraph(File dir, boolean nullLabels)
101+
public DiskDGraph(File dbFile, boolean nullLabels)
96102
{
97103
this.nullLabels = nullLabels;
98104

99-
dir.mkdirs();
100-
File dbFile = new File(dir, "graph."+id+".db");
101-
102105
db = DBMaker.fileDB(dbFile).make();
103106

104107
labels = nullLabels ? null : db.indexTreeList("labels", Serializer.STRING).createOrOpen();
105108

106109
in = db.indexTreeList("in", new SerializerIntList()).createOrOpen();
107110
out = db.indexTreeList("out", new SerializerIntList()).createOrOpen();
111+
112+
if(!nullLabels && labels.size() != in.size())
113+
throw new IllegalStateException("labels list has size "+ labels.size() + ", should be " + in.size() + ".");
114+
115+
if(db.exists("numLinks"))
116+
numLinks = db.atomicInteger("numLinks").createOrOpen().get();
117+
else
118+
for(List<Integer> list : in)
119+
numLinks += list.size();
108120
}
109121

122+
110123
@Override
111124
public int size()
112125
{
@@ -163,7 +176,7 @@ public void remove()
163176

164177
for(int i : series(in.size()))
165178
{
166-
List<Integer> neighbors = in.get(i);
179+
List<Integer> neighbors = new ArrayList<Integer>(in.get(i));
167180

168181
Iterator<Integer> it = neighbors.iterator();
169182
while(it.hasNext())
@@ -174,7 +187,7 @@ public void remove()
174187
}
175188
for(int i : series(out.size()))
176189
{
177-
List<Integer> neighbors = out.get(i);
190+
List<Integer> neighbors = new ArrayList<Integer>(out.get(i));
178191

179192
Iterator<Integer> it = neighbors.iterator();
180193
while(it.hasNext())
@@ -188,7 +201,7 @@ public void remove()
188201
// is higher than the one we just removed.
189202
for(int i : series(in.size()))
190203
{
191-
List<Integer> neighbors = in.get(i);
204+
List<Integer> neighbors = new ArrayList<Integer>(in.get(i));
192205

193206
for(int j : series(neighbors.size()))
194207
{
@@ -201,7 +214,7 @@ public void remove()
201214
}
202215
for(int i : series(out.size()))
203216
{
204-
List<Integer> neighbors = out.get(i);
217+
List<Integer> neighbors = new ArrayList<Integer>(out.get(i));
205218

206219
for(int j : series(neighbors.size()))
207220
{
@@ -390,21 +403,21 @@ public void disconnect(Node<String> other)
390403

391404
int links = 0;
392405

393-
List<Integer> myOut = out.get(mine);
406+
List<Integer> myOut = new ArrayList<Integer>(out.get(mine));
394407
while(myOut.remove((Integer)his))
395408
links++;
396409
out.set(mine, myOut);
397410

398-
List<Integer> hisOut = out.get(his);
411+
List<Integer> hisOut = new ArrayList<Integer>(out.get(his));
399412
while(hisOut.remove((Integer)mine))
400413
links++;
401414
out.set(his, hisOut);
402415

403-
List<Integer> myIn = in.get(mine);
416+
List<Integer> myIn = new ArrayList<Integer>(in.get(mine));
404417
while(myIn.remove((Integer)his));
405418
in.set(mine, myIn);
406419

407-
List<Integer> hisIn = in.get(his);
420+
List<Integer> hisIn = new ArrayList<Integer>(in.get(his));
408421
while(hisIn.remove((Integer)mine));
409422
in.set(his, hisIn);
410423

@@ -651,8 +664,14 @@ public Graph<String> graph()
651664
public void remove()
652665
{
653666
check();
654-
in.get(to.index()).remove((Integer)from.index());
655-
out.get(from.index()).remove((Integer)to.index());
667+
668+
List<Integer> list = new ArrayList<Integer>(in.get(to.index()));
669+
list.remove((Integer)from.index());
670+
in.set(to.index(), list);
671+
672+
list = new ArrayList<Integer>(out.get(from.index()));
673+
list.remove((Integer)to.index());
674+
out.set(from.index(), list);
656675

657676
modCount++;
658677
dead = true;
@@ -1155,6 +1174,28 @@ public List<DNode<String>> neighborsFast(Node<String> node)
11551174
return new NodeList(indices);
11561175
}
11571176

1177+
/**
1178+
* Loads a previous converted graph.
1179+
*
1180+
* @param dbFile
1181+
* @return
1182+
* @throws IOException
1183+
*/
1184+
public static DiskDGraph fromDB(File dbFile)
1185+
throws IOException
1186+
{
1187+
DB db = DBMaker.fileDB(dbFile).make();
1188+
1189+
if(db.exists("labels"))
1190+
{
1191+
db.close();
1192+
return new DiskDGraph(dbFile, false);
1193+
}
1194+
1195+
db.close();
1196+
return new DiskDGraph(dbFile, true);
1197+
}
1198+
11581199
/**
11591200
* Reads a (large) edgelist-encoded file into a DiskDGraph.
11601201
*
@@ -1165,16 +1206,23 @@ public List<DNode<String>> neighborsFast(Node<String> node)
11651206
public static DiskDGraph fromFile(File file, File dir)
11661207
throws IOException
11671208
{
1168-
DiskDGraph graph = new DiskDGraph(dir, true);
1209+
int id = (new Random()).nextInt(10000000);
1210+
1211+
return fromFile(file, dir, new File("graph."+id+".db"));
1212+
}
1213+
public static DiskDGraph fromFile(File file, File tmpDir, File dbFile)
1214+
throws IOException
1215+
{
1216+
DiskDGraph graph = new DiskDGraph(dbFile, true);
11691217

11701218
// * sort the input file by first element
1171-
File forward = new File(dir, "forward.edgelist");
1219+
File forward = new File(tmpDir, "forward.edgelist");
11721220

11731221

11741222
List<File> files = ExternalSort.sortInBatch(
11751223
file,
11761224
new LComp(true), ExternalSort.DEFAULTMAXTEMPFILES,
1177-
Charset.defaultCharset(), dir, false);
1225+
Charset.defaultCharset(), tmpDir, false);
11781226
ExternalSort.mergeSortedFiles(files, forward, new LComp(true), Charset.defaultCharset());
11791227

11801228
System.out.println("Forward sort finished");
@@ -1184,12 +1232,12 @@ public static DiskDGraph fromFile(File file, File dir)
11841232
System.out.println("Forward list read");
11851233

11861234
forward.delete();
1187-
File backward = new File(dir, "backward.edgelist");
1235+
File backward = new File(tmpDir, "backward.edgelist");
11881236

11891237
files = ExternalSort.sortInBatch(
11901238
file,
11911239
new LComp(false), ExternalSort.DEFAULTMAXTEMPFILES,
1192-
Charset.defaultCharset(), dir, false);
1240+
Charset.defaultCharset(), tmpDir, false);
11931241
ExternalSort.mergeSortedFiles(files, backward, new LComp(false), Charset.defaultCharset());
11941242

11951243
System.out.println("Backward sort finished");
@@ -1216,6 +1264,12 @@ public static DiskDGraph fromFile(File file, File dir)
12161264
return graph;
12171265
}
12181266

1267+
public void close()
1268+
{
1269+
db.atomicInteger("numLinks").createOrOpen().set(numLinks);
1270+
db.close();
1271+
}
1272+
12191273
private static long readSorted(List<List<Integer>> list, File file, boolean forward)
12201274
throws IOException
12211275
{

0 commit comments

Comments
 (0)