Expand swap capability and reorganize package

Merges #6

* Use roaring bitset for larger set
* Add tests for roaring bitset, rename "hash table" to "bitset" for correctness

.travis.yml

@@ -11,6 +11,8 @@ script:
   - python setup.py build
   - pip install .
   - pytest tests/
-  - g++ tests/test_hash_table.cpp xswap/hash_table.cpp -o tests/test_hash_table.o -std=c++11
-  - ./tests/test_hash_table.o
+  - g++ tests/test_bitset.cpp xswap/src/bitset.cpp xswap/lib/roaring.c -o tests/test_bitset.o -std=c++11
+  - ./tests/test_bitset.o
+  - g++ tests/test_roaring.cpp xswap/lib/roaring.c xswap/src/bitset.cpp -o tests/test_roaring.o -std=c++11
+  - ./tests/test_roaring.o
   - cat tests/permutation_stats.txt

setup.py

@@ -20,7 +20,7 @@
 
 xswap_cpp_extension = setuptools.Extension(
     'xswap._xswap_backend',
-    sources=['xswap/xswap_wrapper.cpp', 'xswap/hash_table.cpp', 'xswap/xswap.cpp'],
+    sources=['xswap/src/xswap_wrapper.cpp', 'xswap/src/bitset.cpp', 'xswap/src/xswap.cpp', 'xswap/lib/roaring.c'],
     extra_compile_args=["-std=c++11"],
 )
 
@@ -42,5 +42,5 @@
     python_requires='>=3.5',
 
     ext_modules=[xswap_cpp_extension],
-    packages=['xswap'],
+    packages=setuptools.find_packages(),
 )

tests/test_hash_table.cpp → tests/test_bitset.cpp

@@ -2,7 +2,7 @@
 #include <cstdio>
 #include <iostream>
 #include <stdexcept>
-#include "../xswap/xswap.h"
+#include "../xswap/src/xswap.h"
 
 void handle_eptr(std::exception_ptr eptr) {
     try {
@@ -14,7 +14,7 @@ void handle_eptr(std::exception_ptr eptr) {
     }
 }
 
-bool test_add(EdgeHashTable edges_set) {
+bool test_add(UncompressedBitSet edges_set) {
     int edge_to_add[2] = {1, 1};
     edges_set.add(edge_to_add);
     int** fake_edges = (int**)malloc(sizeof(int*) * 16);
@@ -45,7 +45,7 @@ bool test_add(EdgeHashTable edges_set) {
     }
 }
 
-bool test_remove(EdgeHashTable edges_set) {
+bool test_remove(UncompressedBitSet edges_set) {
     int edge_to_add[2] = {1, 1};
     edges_set.add(edge_to_add);
     bool was_added = edges_set.contains(edge_to_add);
@@ -59,7 +59,7 @@ bool test_remove(EdgeHashTable edges_set) {
     return passed;
 }
 
-bool test_oob_insert(EdgeHashTable edges_set) {
+bool test_oob_insert(UncompressedBitSet edges_set) {
 
     int edge_to_add[2] = {4, 4};
     std::exception_ptr eptr;
@@ -76,7 +76,7 @@ bool test_oob_insert(EdgeHashTable edges_set) {
     return false;
 }
 
-bool test_oob_access(EdgeHashTable edges_set) {
+bool test_oob_access(UncompressedBitSet edges_set) {
     int edge_to_access[2] = {4, 4};
     std::exception_ptr eptr;
     try {
@@ -92,7 +92,7 @@ bool test_oob_access(EdgeHashTable edges_set) {
     return false;
 }
 
-bool test_oob_remove(EdgeHashTable edges_set) {
+bool test_oob_remove(UncompressedBitSet edges_set) {
     int edge_to_access[2] = {4, 4};
     std::exception_ptr eptr;
     try {
@@ -108,7 +108,7 @@ bool test_oob_remove(EdgeHashTable edges_set) {
     return false;
 }
 
-bool test_remove_nonexistent(EdgeHashTable edges_set) {
+bool test_remove_nonexistent(UncompressedBitSet edges_set) {
     int edge_to_access[2] = {2, 2};
     std::exception_ptr eptr;
     try {
@@ -124,7 +124,7 @@ bool test_remove_nonexistent(EdgeHashTable edges_set) {
     return false;
 }
 
-bool test_insert_existing(EdgeHashTable edges_set) {
+bool test_insert_existing(UncompressedBitSet edges_set) {
     int edge_to_access[2] = {2, 2};
     edges_set.add(edge_to_access);
     std::exception_ptr eptr;
@@ -145,16 +145,16 @@ main(int argc, char const *argv[]) {
     int num_tests = 7;
     bool test_passed[num_tests];
 
-    EdgeHashTable edges_set = EdgeHashTable(3, 3);
+    UncompressedBitSet edges_set = UncompressedBitSet(3, 3);
     test_passed[0] = test_add(edges_set);
-    edges_set = EdgeHashTable(3, 3);  // Reset so functions don't interfere
+    edges_set = UncompressedBitSet(3, 3);  // Reset so functions don't interfere
     test_passed[1] = test_remove(edges_set);
     test_passed[2] = test_oob_insert(edges_set);
     test_passed[3] = test_oob_access(edges_set);
     test_passed[4] = test_oob_remove(edges_set);
-    edges_set = EdgeHashTable(3, 3);
+    edges_set = UncompressedBitSet(3, 3);
     test_passed[5] = test_remove_nonexistent(edges_set);
-    edges_set = EdgeHashTable(3, 3);
+    edges_set = UncompressedBitSet(3, 3);
     test_passed[6] = test_insert_existing(edges_set);
 
     bool all_tests_passed = true;
@@ -169,5 +169,5 @@ main(int argc, char const *argv[]) {
         std::printf("Test failure\n");
         return 1;
     }
-    edges_set.free_table();
+    edges_set.free_array();
 }

tests/test_roaring.cpp

@@ -0,0 +1,69 @@
+#include <iostream>
+#include "../xswap/src/xswap.h"
+
+
+main(int argc, char const *argv[])
+{
+    int counter, incorrect_contains, incorrect_doesnt_contain;
+
+    // Create real edges to be added to the Roaring set
+    int** real_edges = (int**)malloc(sizeof(int*) * 16);
+    counter = 0;
+    for (int i = 4; i < 8; i++) {
+        for (int j = 4; j < 8; j++) {
+            real_edges[counter] = (int*)malloc(sizeof(int) * 2);
+            real_edges[counter][0] = i;
+            real_edges[counter][1] = j;
+            counter += 1;
+        }
+    }
+
+    Edges edges;
+    edges.edge_array = real_edges;
+    edges.num_edges = 16;
+    RoaringBitSet edges_set = RoaringBitSet(edges);
+
+    // Check that edges added at the creation of the set are contained
+    incorrect_doesnt_contain = 0;
+    for (int i = 4; i < 8; i++) {
+        for (int j = 4; j < 8; j++) {
+            int edge[2] = {i, j};
+            if (!edges_set.contains(edge)) {
+                incorrect_doesnt_contain += 1;
+            }
+        }
+    }
+
+    // Create fake edges and check that they are not in the set
+    counter = 0;
+    incorrect_contains = 0;
+    for (int i = 0; i < 4; i++) {
+        for (int j = 0; j < 4; j++) {
+            int fake_edge[2] = {i, j};
+            // Check that this edge is not in the set
+            if (edges_set.contains(fake_edge)) {
+                incorrect_contains += 1;
+            }
+            // Add the edge and check that it was added
+            edges_set.add(fake_edge);
+            if (!edges_set.contains(fake_edge)) {
+                incorrect_doesnt_contain += 1;
+            }
+            // Remove the edge and check that it is removed
+            edges_set.remove(fake_edge);
+            if (edges_set.contains(fake_edge)) {
+                incorrect_contains += 1;
+            }
+            counter += 1;
+        }
+    }
+
+    free(real_edges);
+    if (incorrect_contains == 0 && incorrect_doesnt_contain == 0) {
+        std::cout << "All tests passed" << "\n";
+        return 0;
+    } else {
+        std::cout << "Tests failed " << incorrect_contains << " " << incorrect_doesnt_contain << "\n";
+        return 1;
+    }
+}

tests/test_time.py

@@ -10,7 +10,7 @@
 
 def load_edges():
     edges_url = "https://github.com/greenelab/xswap/raw/{}/{}".format(
-        "51cad9392880db08a4c5870f08a670c52877e78f", "graphs/GiG_edges.txt")
+        "8c31b4cbdbbf2cfa5018b1277bbd0e9f6263e573", "graphs/GiG_edges_reduced.txt")
     response = requests.get(edges_url)
     edges = list()
     for edge in response.iter_lines():
@@ -25,9 +25,10 @@ def test_time():
     t1 = time.time()
     new_edges, stats = xswap.permute_edge_list(edges)
     t2 = time.time()
-    print(str(t2 - t1) + "  seconds elapsed.")
+    time_diff = t2 - t1
+    print("{:.4f}  seconds elapsed.".format(time_diff))
     assert edges != new_edges
-    assert t2 - t1 < 5
+    assert time_diff < 5
 
     num_repeats = 0
     old_set = set(edges)
@@ -38,4 +39,4 @@ def test_time():
     p_unch = num_repeats / len(edges)
     with open(test_directory + 'permutation_stats.txt', 'w') as f:
         f.write('Runtime: {:.3f} sec. {:.3f} percent unchanged of {} total edges after '
-                '{} swap attempts\n'.format(t2 - t1, p_unch, len(edges), 10*len(edges)))
+                '{} swap attempts\n'.format(time_diff, p_unch, len(edges), 10*len(edges)))

xswap/__init__.py

@@ -1,5 +1,11 @@
-from .xswap import permute_edge_list
+from xswap.xswap import permute_edge_list
+from xswap import preprocessing
 
-__version__ = '0.0.1'
+__version__ = '0.0.2'
 
-__all__ = [permute_edge_list]
+__all__ = [
+    permute_edge_list,
+    preprocessing.load_str_edges,
+    preprocessing.load_processed_edges,
+    preprocessing.map_str_edges,
+]