Merge branch 'bmyerz-clang-groupby' into bmyerz-merge-shuffle-groupby

Conflicts:
	examples/grappalog.sh
	raco/clang.py
	raco/grappa_templates/base_query.template
	raco/grappalang.py

c_test_environment/.gitignore

@@ -19,4 +19,9 @@ tmp/
 Index
 Str
 *.ttl
-log.rb
+edges
+*.ps
+*.pdf
+log*.rb
+*.cpp.*
+test.txt

c_test_environment/clang_tests.py

@@ -1,137 +1,30 @@
 import unittest
 from testquery import checkquery
 from testquery import testdbname
+from testquery import ClangRunner
 from generate_test_relations import generate_default
+from generate_test_relations import need_generate
 from raco.language import CCAlgebra
+from platform_tests import PlatformTest
 
 import sys
 import os
 sys.path.append('./examples')
 from emitcode import emitCode
 from osutils import Chdir
 
-# skipping
-from nose.tools import nottest
 
-
-class ClangTest(unittest.TestCase):
+class ClangTest(unittest.TestCase, PlatformTest):
     def check(self, query, name):
-        chdir = Chdir("c_test_environment")
-        emitCode(query, name, CCAlgebra)
-        checkquery(name)
+        with Chdir("c_test_environment") as d:
+            emitCode(query, name, CCAlgebra)
+            checkquery(name, ClangRunner())
 
     def setUp(self):
-        chdir = Chdir("c_test_environment")
-        if not os.path.isfile(testdbname()):
-            generate_default()  
+        with Chdir("c_test_environment") as d:
+          if need_generate():
+                generate_default()  
 
-    # @nottest: excluding these tests from nosetests
-    # Currently running clang_tests in the hosted travis.ci environment fails
-    # to run the compiled C++ programs. Exit code -4 and no output on stderr/stdout.
-    # Run these tests separately with `python clang_tests.py` from the root of datalogcompiler/
-    @nottest
-    def test_scan(self):
-        self.check("A(s1) :- T1(s1)", "scan")
-
-    @nottest
-    def test_select(self):
-        self.check("A(s1) :- T1(s1), s1>5", "select") 
-
-    @nottest
-    def test_join(self):
-        self.check("A(s1,o2) :- T3(s1,p1,o1), R3(o2,p1,o2)", "join")
-
-    @nottest
-    def test_select_conjunction(self):
-        self.check("A(s1) :- T1(s1), s1>0, s1<10", "select_conjunction")
-
-    @nottest
-    def test_two_var_select(self):
-        self.check("A(s1,s2) :- T2(s1,s2), s1<9, s2<9", "two_var_select")
-
-    @nottest
-    def test_self_join(self):
-        self.check("A(a,b) :- R2(a,b), R2(a,c)", "self_join")
-
-    @nottest
-    def test_two_path(self):
-        self.check("A(a,b,c) :- R2(a,b), S2(b,c)", "two_path")
-
-    @nottest
-    def test_two_hop(self):
-        self.check("A(a,c) :- R2(a,b), S2(b,c)", "two_hop")
-
-    @nottest
-    def test_three_path(self):
-        self.check("A(a,b,c) :- R2(a,b), S2(b,c), T2(c,d)", "three_path")
-
-    @nottest
-    def test_self_three_path(self):
-        self.check("A(a,b,c) :- R2(a,b), R2(b,c), R2(c,d)", "self_three_path"),
-
-    @nottest
-    def test_directed_triangles(self):
-        self.check("A(a,b,c) :- R2(a,b), S2(b,c), T2(c,a)", "directed_triangles"),
-
-    @nottest
-    def test_directed_squares(self):
-        self.check("A(a,b,c,d) :- R2(a,b), S2(b,c), T2(c,d), R3(d,a,x)", "directed_squares"),
-
-    @nottest
-    def test_select_then_join(self):
-        self.check("A(s1,s2,s3) :- T3(s1,s2,s3), R2(s3,s4), s1<s2, s4<9", "select_then_join"),
-
-    # TODO: All unions are currently treated as unionAll
-    @nottest
-    def test_union(self):
-        self.check("""A(s1) :- T1(s1)
-    A(s1) :- R1(s1)""", "union")
-
-    @nottest
-    def test_swap(self):
-        self.check("A(y,x) :- R2(x,y)", "swap"),
-
-    @nottest
-    def test_apply(self):
-        self.check("""A(x,y) :- T2(x,y)
-    B(a) :- A(z,a)""", "apply")
-
-    @nottest
-    def test_apply_and_self_join(self):
-        self.check("""A(x,z) :- T3(x,y,z), y < 4
-    B(x,t) :- A(x,z), A(z,t)""", "apply_and_self_join")
-
-    @nottest
-    def test_union_apply_and_self_join(self):
-         self.check("""A(x,y) :- T2(x,y), R1(x), y < 4
-            A(x,y) :- R2(x,y), T1(x)
-    B(x,z,t) :- A(x,z), A(z,t)""", "union_apply_and_self_join")
-
-    @nottest
-    def test_union_of_join(self):
-         self.check("""A(s1,s2) :- T2(s1,s2)
-    A(s1,s2) :- R2(s1,s3), T2(s3,s2)""", "union_of_join")
-
-    @nottest
-    def test_union_then_join(self):
-         self.check("""A(s1,s2) :- T2(s1,s2)
-    A(s1,s2) :- R2(s1,s2)
-    B(s1) :- A(s1,s2), S1(s1)""", "union_then_join")
-
-    @nottest
-    def test_join_of_two_unions(self):
-        self.check("""A(s1,s2) :- T2(s1,s2)
-    A(s1,s2) :- R2(s1,s2)
-    B(s1) :- A(s1,s2), A(s1,s3)""", "join_of_two_unions")
-
-    @nottest
-    def test_join_swap_indexing(self):
-        self.check("""A(a,h,y) :- T3(a,b,c), R3(x, y, z), S3(g,h,j), z=c, j=x""", "join_swap_indexing")
-
-    @nottest
-    def test_head_scalar_op(self):
-        self.check("""A(a+b) :- R2(a,b)""", "head_scalar_op")
-
 
 if __name__ == '__main__':
     unittest.main()

c_test_environment/generate_test_relations.py

@@ -2,9 +2,13 @@
 import os
 from subprocess import check_call
 
+def get_name(basename, fields):
+    return basename+str(fields)
+
 def generate(basename, fields, tuples, datarange):
-    with open(basename+str(fields), 'w') as f:
-        print "generating %s" % (os.path.abspath(basename+str(fields)))
+    fn = get_name(basename, fields)
+    with open(fn, 'w') as f:
+        print "generating %s" % (os.path.abspath(fn))
         for i in range(0,tuples):
             for j in range(0,fields):
                 dat = random.randint(0, datarange)
@@ -30,19 +34,33 @@ def importStatement(basename, fields):
     text = template % locals()
     return text
 
-def generate_default():
+def need_generate(cpdir=None):
+  if cpdir:
+    return not os.path.isfile(os.path.join(cpdir, 'test.db'))
+  else:
+    return not os.path.isfile('test.db')
+
+def generate_default(cpdir=None):
     print 'generating'
     with open('importTestData.sql', 'w') as f:
         for n in ['R','S','T']:
             for nf in [1,2,3]:
                 generate(n, nf, 30, 10)
                 f.write(importStatement(n, nf))
+                if cpdir:
+                    fn = get_name(n, nf)
+                    check_call(['ln', '-fs', os.path.abspath(fn), cpdir])
+
+        if cpdir:
+          check_call(['ln', '-fs', os.path.abspath('test.db'), cpdir])
 
     # import to sqlite3
     print 'importing'
+    check_call(['rm', '-f', 'test.db'])
     with open('importTestData.sql', 'r') as f:
        check_call(['sqlite3', 'test.db'], stdin=f) 
 
+
 if __name__ == "__main__":
     generate_default()
 

c_test_environment/grappalang_tests.py

@@ -0,0 +1,30 @@
+import unittest
+from testquery import checkquery
+from testquery import GrappalangRunner
+from generate_test_relations import generate_default
+from generate_test_relations import need_generate
+from raco.language import GrappaAlgebra
+from platform_tests import PlatformTest
+
+import sys
+import os
+sys.path.append('./examples')
+from emitcode import emitCode
+from osutils import Chdir
+
+
+class GrappaTest(unittest.TestCase, PlatformTest):
+    def check(self, query, name):
+        with Chdir("c_test_environment") as d:
+            emitCode(query, 'grappa_%s' % name, GrappaAlgebra)
+            checkquery(name, GrappalangRunner())
+
+    def setUp(self):
+        with Chdir("c_test_environment") as d:
+            targetpath = os.path.join(os.environ.copy()['GRAPPA_HOME'], 'build/Make+Release/applications/join')
+            if need_generate(targetpath):
+              generate_default(targetpath)
+
+
+if __name__ == '__main__':
+    unittest.main()

c_test_environment/hash.h

@@ -5,8 +5,8 @@
 
 #include <iostream>
 
-template <typename T>
-void insert(std::unordered_map<int64_t, std::vector<T>* >& hash, T tuple, uint64_t pos) {
+template <typename T, typename K>
+void insert(std::unordered_map<K, std::vector<T>* >& hash, T tuple, uint64_t pos) {
   auto key = tuple.get(pos);
   auto r = hash.find(key);
   if (r != hash.end()) {
@@ -19,8 +19,39 @@ void insert(std::unordered_map<int64_t, std::vector<T>* >& hash, T tuple, uint64
   }
 }
 
-template <typename T>
-std::vector<T>& lookup(std::unordered_map<int64_t, std::vector<T>* >& hash, int64_t key) {
+template <typename T, typename K, typename V>
+void SUM_insert(std::unordered_map<K, V >& hash, T tuple, uint64_t keypos, uint64_t valpos) {
+  auto key = tuple.get(keypos);
+  auto val = tuple.get(valpos);
+  // NOTE: this method is only valid for 0 identity functions
+  auto& slot = hash[key];
+  slot += val;
+}
+
+template <typename T, typename K, typename V>
+void COUNT_insert(std::unordered_map<K, V>& hash, T tuple, uint64_t keypos, uint64_t valpos) {
+  auto key = tuple.get(keypos);
+  auto val = tuple.get(valpos);
+  // NOTE: this method is only valid for 0 identity functions
+  auto& slot = hash[key];
+  slot += 1;
+}
+
+// one key
+template <typename T, typename V>
+void SUM_insert(V& var, T tuple, uint64_t valpos) {
+  auto val = tuple.get(valpos);
+  var += val;
+}
+
+// one key
+template <typename T, typename V>
+void COUNT_insert(V& var, T tuple, uint64_t valpos) {
+  var += 1;
+}
+
+template <typename T, typename K>
+std::vector<T>& lookup(std::unordered_map<K, std::vector<T>* >& hash, K key) {
   static std::vector<T> emptyResult;
 
   auto r = hash.find(key);

c_test_environment/osutils.py

@@ -7,11 +7,14 @@ class Chdir:
   Safe cd that is really a pushd then popd on leaving the scope
   """
   def __init__( self, newPath ):  
-    self.savedPath = os.getcwd()
-    os.chdir(newPath)
+    self._newPath = newPath
 
-  def __del__( self ):
-    os.chdir( self.savedPath )
+  def __enter__( self ):
+    self._savedPath = os.getcwd()
+    os.chdir(self._newPath)
+
+  def __exit__( self, x, y, z ):
+    os.chdir( self._savedPath )
 
 def mkdir_p(path):
     try: