Merge pull request #47 from cosmo-ethz/slice2d

Extended slicing and reference to member vars

HISTORY.rst

@@ -3,6 +3,14 @@
 History
 -------
 
+0.x.x (2016-0x-xx)
+++++++++++++++++++
+
+* Fixed bug in 2d array slicing
+* Array slicing with negative index
+* Fixed name clash bug with object attributes
+* Replaced assignment with reference to object attributes 
+
 0.5.0 (2016-01-20)
 ++++++++++++++++++
 

examples/hope_arrays.py

@@ -43,6 +43,9 @@ def array_slicing_2D(a):
     a[:, 1:] = 2
     a[2:, 2:] = 4
     a[3, 3] = 5
+    a[1, :-2] = 6
+    a[:-2, 1] = 7
+    a[0, -1] = 8
     return a
 
 @hope.jit
@@ -63,7 +66,7 @@ def example():
     a = np.zeros((4,4))
     r = array_slicing_2D(a)
     print(r)
-    assert np.sum(r) == 39
+    assert np.sum(r) == 60
 
     #assign a slice of an array to a slice
     a = np.zeros(4)

hope/_ast.py

@@ -105,6 +105,13 @@ def __init__(self, parent, name, dtype, dims, char):
     def __eq__(self, other):
         return isinstance(other, ObjectAttr) and self.parent == other.parent and self.name == other.name and self.dtype == other.dtype and self.shape == other.shape and self.char == other.char
 
+    def getTrace(self):
+        parent = self.parent
+        trace = [self.name]
+        while not parent is None:
+            trace.insert(0, parent.name)
+            parent = parent.parent
+        return trace
 
 class Dimension(Token):
     def __init__(self, variable, dim):
@@ -115,6 +122,10 @@ def __eq__(self, other):
             and isinstance(self.variable, Variable) and self.variable.name == other.variable.name and self.variable.dtype == other.variable.dtype \
             and len(self.variable.shape) == len(other.variable.shape) and self.variable.scope == other.variable.scope
 
+class DimensionSlice(Dimension):
+    def __init__(self, variable, dim, slice):
+        super(DimensionSlice, self).__init__(variable, dim)
+        self.slice = slice
 
 class View(Token):
     def __init__(self, variable, extents):
@@ -127,13 +138,16 @@ def __init__(self, variable, extents):
         if len(variable.shape) != len(extents):
             raise Exception("Extends of variable and subscript do not match")
         self.variable, self.dtype, self.extents, self.shape = variable, variable.dtype, extents, []
-        for variable_extent, extent in zip(self.variable.shape, extents):
+        for ind, (variable_extent, extent) in enumerate(zip(self.variable.shape, extents[:])):
             # TODO: check if variable extends and extends do match
             if isinstance(extent, tuple):
                 lower, upper = extent
                 if lower is None: lower = variable_extent[0]
                 if isinstance(lower, Number) and lower.value == 0: lower = None
                 if upper is None: upper = variable_extent[1]
+                if isinstance(upper, Number) and upper.value < 0: 
+                    upper = DimensionSlice(variable, variable_extent[1].dim, upper)
+                    extents[ind] = (lower, copy.deepcopy(upper))
                 self.shape.append((lower, upper))
 
     def __eq__(self, other):
@@ -340,6 +354,12 @@ class Block(Token):
     def __init__(self, expr):
         self.body, self.shape, self.merged = [expr], expr.shape, None
 
+class Reference(Token):
+    def __init__(self, target, value):
+        self.target, self.value, self.dtype, self.shape = target, value, target.dtype, []
+
+    def __eq__(self, other):
+        return isinstance(other, Reference) and self.target == other.target and self.value == other.value
 
 class Body(Token):
     def __init__(self, blocks):

hope/_dump.py

@@ -56,6 +56,9 @@ def visit_Dimension(self, node):
             return "{0}@{1}".format(node.variable.name, node.dim)
         else:
             raise Exception("Unknown type {0}".format(node.variable.name))
+
+    def visit_DimensionSlice(self, node):
+        return "{0}+{1}".format(self.visit_Dimension(node), self.visit(node.slice))
 
     def visit_View(self, node):
         ret = node.variable.name

hope/_generator.py

@@ -24,14 +24,14 @@ def __init__(self):
     def getVariableExtent(self, node):
         extent = ""
         for ind, segment in enumerate(node.shape):
-            segmentstr = "{0}:{1}".format("" if segment[0] is None else self.dumper.visit(segment[0]), self.dumper.visit(segment[1]))
+            segmentstr = self.get_segmentstr(*segment)
             if not segment[0] is None:
                 raise Exception("Variable slices needs to start with None: {0}".format(node.name))
             if not segmentstr in self.merged:
                 raise Exception("Unknown slice {0} in variable {1}".format(segmentstr, node.name))
             if ind > 0:
                 extent = "({0})*{1}".format(extent, self.visit(segment[1]))
-            extent += "{0}{1}".format(" + " if len(extent) > 0 else "", self.slicemap[self.merged[segmentstr]])
+            extent += "{0}{1}".format(" + " if len(extent) > 0 else "", self.slicemap[self.get_slicemap_key(ind, *segment)])
         return extent
 
     def visit_Number(self, node):
@@ -49,11 +49,7 @@ def visit_Variable(self, node):
             return "c{0}[{1}]".format(node.name, self.getVariableExtent(node));
 
     def visit_ObjectAttr(self, node):
-        parent = node.parent
-        trace = [node.name]
-        while not parent is None:
-            trace.insert(0, parent.name)
-            parent = parent.parent
+        trace = node.getTrace()
         if len(node.shape) == 0:
             return "c" + ".c".join(trace)
         else:
@@ -72,8 +68,12 @@ def visit_Dimension(self, node):
         else:
             raise Exception("Unknown type {0}".format(node.variable.name))
 
+    def visit_DimensionSlice(self, node):
+        return "{0}+{1}".format(self.visit_Dimension(node), self.visit(node.slice))
+
     def visit_View(self, node):
         subscript = ""
+        extent_ind = 0
         for ind, (extent, segment) in enumerate(zip(node.extents, node.variable.shape)):
             if ind > 0:
                 subscript = "(int)({0})*{1}".format(subscript, self.visit(segment[1]))
@@ -84,10 +84,15 @@ def visit_View(self, node):
                 if isinstance(lower, Number) and lower.value == 0: lower = None
                 if upper is None: upper = segment[1]
                 seg = "{0} + ".format(self.visit(lower)) if not lower is None else ""
-                seg += self.slicemap[self.merged["{0}:{1}".format("" if lower is None else self.dumper.visit(lower), self.dumper.visit(upper))]]
+                key = self.get_slicemap_key(extent_ind, lower, upper)
+                seg += self.slicemap[key]
+#                 seg += self.slicemap[self.merged["{0}:{1}".format("" if lower is None else self.dumper.visit(lower), self.dumper.visit(upper))]]
                 segstr = "{0}:{1}".format("" if lower is None else self.dumper.visit(lower), self.dumper.visit(upper))
+                extent_ind += 1
             else:
                 seg = self.visit(extent)
+                if isinstance(extent, Number) and extent.value < 0:
+                    seg = "{0}+{1}".format(self.visit_Dimension(segment[1]), seg)
                 segstr = self.dumper.visit(extent)
             if config.rangecheck:
                 subscript += "native_rangecheck({0}".format(seg)
@@ -111,21 +116,23 @@ def visit_NumpyContraction(self, node):
             raise Exception("Only the numpy.sum contraction is implemented!")
         if len(node.value.shape):
             ret = "{0} = ({1})0;\n".format(self.visit(node.variable), PY_C_TYPE[node.dtype])
+            keys = []
             for ind, segment in enumerate(node.value.shape):
                 ret += "{0}for (npy_intp i{1} = 0; i{1} < {2} - {3}; ++i{1}) {{\n".format( \
                       "\t" * ind \
                     , self.next_loopid \
                     , self.visit(segment[1]) \
                     , 0 if segment[0] is None else self.visit(segment[0]) \
                 )
-                segmentstr = "{0}:{1}".format("" if segment[0] is None else self.dumper.visit(segment[0]), self.dumper.visit(segment[1]))
+                segmentstr = self.get_segmentstr(*segment)
                 if not segmentstr in self.merged:
                     self.merged[segmentstr] = segmentstr
-                self.slicemap[self.merged[segmentstr]] = "i{0}".format(self.next_loopid)
+                keys.append(self.get_slicemap_key(ind, *segment))
+                self.slicemap[keys[-1]] = "i{0}".format(self.next_loopid)
                 self.next_loopid += 1
             ret += "{0}{1} += {2};".format("\t" * len(node.value.shape), self.visit(node.variable), self.visit(node.value))
-            for ind, segment in enumerate(node.value.shape):
-                del self.slicemap[self.merged["{0}:{1}".format("" if segment[0] is None else self.dumper.visit(segment[0]), self.dumper.visit(segment[1]))]]
+            for ind, (key, segment) in enumerate(zip(keys, node.value.shape)):
+                del self.slicemap[key]
                 ret += "\n{0}}}".format("\t" * (len(node.value.shape) - 1 - ind))
             return ret
         else:
@@ -139,6 +146,18 @@ def visit_Assign(self, node):
         else:
             return "{0} = {1};".format(self.visit(node.target), self.visit(node.value))
 
+    def visit_Reference(self, node):
+        target = node.target
+        trace = node.value.getTrace()
+        if isinstance(target, ObjectAttr): # self.x = self.y
+            return "c{0} = c{1};".format(".c".join(target.getTrace()), ".c".join(trace))
+        if len(target.shape) == 0: # [int] x = self.y
+            return "{0} c{1} = c{2};".format(PY_C_TYPE[target.dtype], target.name, ".c".join(trace))
+        else: # [array] x = self.y
+            return "PyObject * p{0} = (PyObject *)PyArray_GETCONTIGUOUS((PyArrayObject *)c{1});\n".format(target.name, ".p".join(trace)) \
+                +  "npy_intp * s{0} = c{1};\n".format(target.name, ".s".join(trace)) \
+                +  "{0} * c{1} = c{2};".format(PY_C_TYPE[target.dtype], target.name, ".c".join(trace))
+
     def visit_AugAssign(self, node):
         if node.op == "**=":
             return "{0} = std::pow({0}, {1});".format(self.visit(node.target), self.visit(node.value))
@@ -299,20 +318,25 @@ def visit_Return(self, node):
     def visit_Block(self, node):
         if len(node.shape):
             ret = "";
+            keys = []
             for ind, segment in enumerate(node.shape):
-                ret += "{0}for (npy_intp i{1} = 0; i{1} < {2} - {3}; ++i{1}) {{\n".format( \
-                      "\t" * ind \
-                    , self.next_loopid \
-                    , self.visit(segment[1]) \
-                    , 0 if segment[0] is None else self.visit(segment[0]) \
-                )
-                self.slicemap[self.merged["{0}:{1}".format("" if segment[0] is None else self.dumper.visit(segment[0]), self.dumper.visit(segment[1]))]] = "i{0}".format(self.next_loopid)
+                ret += "{0}for (npy_intp i{1} = 0; i{1} < {2} - {3}; ++i{1}) {{\n".format(
+                          "\t" * ind, 
+                          self.next_loopid, 
+                          self.visit(segment[1]), 
+                          0 if segment[0] is None else self.visit(segment[0])
+                        )
+                keys.append(self.get_slicemap_key(ind, *segment))
+                self.slicemap[keys[-1]] = "i{0}".format(self.next_loopid)
                 self.next_loopid += 1
+
             ret += "{0}".format("\t" * len(node.shape))
             ret += "\n{0}".format("\t" * len(node.shape)).join("\n".join([self.visit(expr) for expr in node.body]).split("\n"))
-            for ind, segment in enumerate(node.shape):
-                del self.slicemap[self.merged["{0}:{1}".format("" if segment[0] is None else self.dumper.visit(segment[0]), self.dumper.visit(segment[1]))]]
+
+            for ind, key in enumerate(keys):
+                del self.slicemap[key]
                 ret += "\n{0}}}".format("\t" * (len(node.shape) - 1 - ind))
+
             return ret
         else:
             return "\n".join(["{0}".format(self.visit(expr)) for expr in node.body])
@@ -382,7 +406,12 @@ def visit_Module(self, node):
         return "\n".join(list(self.library.values())) + code
 
 
-
+    def get_segmentstr(self, lower, upper):
+        return "{0}:{1}".format("" if lower is None else self.dumper.visit(lower), self.dumper.visit(upper))
+
+    def get_slicemap_key(self, ind, lower, upper):
+        segmentstr = self.get_segmentstr(lower, upper)
+        return "i{0}>{1}".format(ind, self.merged[segmentstr])
 
 def generate(modtoken, localfilename):
     """

hope/_transformer.py

@@ -35,7 +35,13 @@ def visit_Object(self, node):
     def visit_ObjectAttr(self, node):
         return []
     def visit_Dimension(self, node):
+        if isinstance(node.variable, ObjectAttr):
+            trace = node.variable.getTrace()
+            return [".".join(trace)]
+
         return [node.variable.name]
+    def visit_DimensionSlice(self, node):
+        return self.visit_Dimension(node)
     def visit_View(self, node):
         symbols = self.visit(node.variable)
         for upper, lower in node.shape:
@@ -46,6 +52,8 @@ def visit_Expr(self, node):
         return self.visit(node.value)
     def visit_Assign(self, node):
         return list(set(self.visit(node.target) + self.visit(node.value)))
+    def visit_Reference(self, node):
+        return list(set(self.visit(node.target) + self.visit(node.value)))
     def visit_AugAssign(self, node):
         return list(set(self.visit(node.target) + self.visit(node.value)))
     def visit_UnaryOp(self, node):
@@ -284,8 +292,9 @@ def visit_Assign(self, node):
                 self.variables[target.name] = Variable(target.name, copy.deepcopy(value.shape), value.dtype)
                 target = self.variables[target.name]
             elif isinstance(target, Variable) and len(target.shape) == 0: pass
+            elif isinstance(target, ObjectAttr): pass
             elif not isinstance(target, View):
-                raise Exception("Assignments are only allowed to views")
+                raise Exception("Assignments are only allowed to views or variables")
 
             # TODO: should we check dtypes?
             if len(target.shape) > 0 and len(value.shape) == 0: pass
@@ -294,7 +303,19 @@ def visit_Assign(self, node):
 
             else:
                 self.merge_shapes(target, value)
+
+
+            if isinstance(value, ObjectAttr) and not isinstance(target, View): # reference to member
+                name = ".".join(value.getTrace())
+                self.variables[name] = Variable(name, shape=copy.deepcopy(value.shape), dtype=value.dtype, scope="body")
+                if not isinstance(target, ObjectAttr): # avoid that target is allocated
+                    self.variables[target.name].scope = "body"
+                    self.variables[target.name].allocated = True
+
+                return Reference(target, value)
+
             return Assign(target, value)
+
         except Exception as ex:
             from hope._tosource import tosource
             ex.args = ((ex.args[0] if ex.args else "") + "\nin line " + tosource(node),) + ex.args[1:]
@@ -320,7 +341,12 @@ def visit_Subscript(self, node):
         return View(self.visit(node.value), self.visit(node.slice))
 
     def visit_UnaryOp(self, node):
-        return UnaryOp(type(node.op).__name__, self.visit(node.operand))
+        op = type(node.op).__name__
+        operand = self.visit(node.operand)
+        if isinstance(operand, Number):
+            opname, _ = UNARY_OPERATORS[op]
+            return operand if opname == "+" else Number(-operand.value)
+        return UnaryOp(op, operand)
 
     def visit_BinOp(self, node):
         left, right = self.visit(node.left), self.visit(node.right)
@@ -377,6 +403,10 @@ def visit_Slice(self, node):
             raise Exception("Step size other than 1 are not supported")
         lower = node.lower if node.lower is None else self.visit(node.lower)
         upper = node.upper if node.upper is None else self.visit(node.upper)
+        if isinstance(lower, Number) and lower.value < 0:
+            raise UnsupportedFeatureException("Negative slices not supported")
+#         if isinstance(upper, Number) and upper.value < 0:
+#             raise UnsupportedFeatureException("Negative slices not supported")
         return [(lower, upper)]
 
     def visit_ExtSlice(self, node):
@@ -431,7 +461,7 @@ def visit_Attribute(self, node):
                 return Number(np.pi)
             else:
                 return NumpyAttr(node.attr)
-        elif isinstance(node.ctx, ast.Load) and isinstance(node.value, ast.Name) and isinstance(node.value.ctx, ast.Load) \
+        elif (isinstance(node.ctx, ast.Load) or isinstance(node.ctx, ast.Store)) and isinstance(node.value, ast.Name) and isinstance(node.value.ctx, ast.Load) \
                 and node.value.id in self.variables and isinstance(self.variables[node.value.id], Object) \
                 and (node.attr in self.variables[node.value.id].attrs or hasattr(self.variables[node.value.id].instance, node.attr)):
             return self.variables[node.value.id].getAttr(node.attr)