add support for untyped NULL, probably not complete, but tests pass

project-m36.cabal

@@ -403,7 +403,12 @@ Test-Suite test-sql
     type: exitcode-stdio-1.0
     main-is: SQL/InterpreterTest.hs
     Other-Modules: SQL.Interpreter.Select, SQL.Interpreter.Base, TutorialD.Interpreter.Base, TutorialD.Interpreter.RelationalExpr, TutorialD.Interpreter.Types, TutorialD.Interpreter.DatabaseContextExpr, TutorialD.Interpreter.RODatabaseContextOperator, ProjectM36.Interpreter, SQL.Interpreter.CreateTable
-        TutorialD.Printer
+                   TutorialD.Printer,
+                   SQL.Interpreter.DBUpdate,
+                   SQL.Interpreter.Delete,
+                   SQL.Interpreter.DropTable,
+                   SQL.Interpreter.Insert,
+                   SQL.Interpreter.Update
 
     Build-Depends: base, HUnit, Cabal, containers, hashable, unordered-containers, mtl, vector, time, bytestring, uuid, stm, deepseq, deepseq-generics, parallel, cassava, attoparsec, gnuplot, directory, temporary, haskeline, megaparsec, text, base64-bytestring, data-interval, filepath, stm-containers, list-t, project-m36, random, MonadRandom, semigroups, parser-combinators, prettyprinter, scientific, recursion-schemes
 

src/lib/ProjectM36/DataTypes/SQL/Null.hs

@@ -8,6 +8,7 @@ import qualified Data.Vector as V
 import ProjectM36.AtomFunction
 import ProjectM36.Tuple
 import ProjectM36.Relation
+import Data.Maybe (isJust)
 
 -- analogous but not equivalent to a Maybe type due to how NULLs interact with every other value
 
@@ -17,7 +18,10 @@ nullAtomType arg = ConstructedAtomType "SQLNullable" (M.singleton "a" arg)
 nullTypeConstructorMapping :: TypeConstructorMapping
 nullTypeConstructorMapping = [(ADTypeConstructorDef "SQLNullable" ["a"],
                                 [DataConstructorDef "SQLNull" [],
-                                 DataConstructorDef "SQLJust" [DataConstructorDefTypeVarNameArg "a"]])
+                                 DataConstructorDef "SQLJust" [DataConstructorDefTypeVarNameArg "a"]]),
+                               -- used in SQL conversion from in expressions such as INSERT INTO s(city) VALUES (NULL) where the query expression must defer type resolution to SQLNull.
+                               (ADTypeConstructorDef "SQLNullOfUnknownType" [],
+                                [DataConstructorDef "SQLNullOfUnknownType" []])
                               ]
 
 nullAtomFunctions :: AtomFunctions
@@ -119,6 +123,7 @@ coalesceBool _other = Left AtomFunctionTypeMismatchError
 isSQLBool :: Atom -> Bool
 isSQLBool atom = case atomTypeForAtom atom of
                    ConstructedAtomType "SQLNullable" _ -> True
+                   ConstructedAtomType "SQLNullOfUnknownType" _ -> True
                    BoolAtomType -> True
                    _ -> False
 
@@ -131,6 +136,7 @@ sqlBool (ConstructedAtom dConsName aType []) |
   dConsName == "SQLNull" &&
   (aType == nullAtomType BoolAtomType ||
    aType == nullAtomType (TypeVariableType "a")) = Nothing
+sqlBool (ConstructedAtom "SQLNullOfUnknownType" _ []) = Nothing
 sqlBool (BoolAtom tf) = Just tf
 sqlBool x | isSQLBool x = error "internal sqlBool type error" -- should be caught above
 sqlBool other = error ("sqlBool type mismatch: " <> show other)
@@ -173,8 +179,22 @@ isNullOrType aType atom = atomTypeForAtom atom == nullAtomType aType || atomType
 
 isNull :: Atom -> Bool
 isNull (ConstructedAtom "SQLNull" (ConstructedAtomType "SQLNullable" _) []) = True
+isNull (ConstructedAtom "SQLNullOfUnknownType" (ConstructedAtomType "SQLNullOfUnknownType" _) []) = True
 isNull _ = False
 
+isNullAtomType :: AtomType -> Bool
+isNullAtomType = isJust . atomTypeFromSQLNull
+
+atomTypeFromSQLNull :: AtomType -> Maybe AtomType
+atomTypeFromSQLNull (ConstructedAtomType "SQLNullOfUnknownType" _) = Nothing
+atomTypeFromSQLNull (ConstructedAtomType "SQLNullable" vars)
+  | M.size vars == 1 =
+    case M.elems vars of
+      [] -> Nothing
+      [t] -> Just t
+      _ts -> Nothing
+atomTypeFromSQLNull _ = Nothing
+
 sqlIntegerBinaryFunction :: AtomType -> (Integer -> Integer -> Atom) -> [Atom] -> Either AtomFunctionError Atom
 sqlIntegerBinaryFunction expectedAtomType op [a,b] 
   | isNullOrType IntegerAtomType a && isNullOrType IntegerAtomType b = do
@@ -279,3 +299,14 @@ sqlIsNull :: AtomFunctionBodyType
 sqlIsNull [ConstructedAtom "SQLNull" (ConstructedAtomType "SQLNullable" _) []] = pure (BoolAtom True)
 sqlIsNull [_arg] = pure (BoolAtom False)
 sqlIsNull _other = Left AtomFunctionTypeMismatchError
+
+isSQLNullableType :: AtomType -> Bool
+isSQLNullableType (ConstructedAtomType "SQLNullable" _) = True
+isSQLNullableType _ = False
+
+isSQLNullableSpecificType :: AtomType -> AtomType -> Bool
+isSQLNullableSpecificType (ConstructedAtomType "SQLNullable" vars) expectedType | M.elems vars == [expectedType] = True
+isSQLNullableSpecificType _ _ = False
+
+isSQLNullUnknownType :: AtomType -> Bool
+isSQLNullUnknownType t = t == ConstructedAtomType "SQLNullOfUnknownType" mempty

src/lib/ProjectM36/RelationalExpression.hs

@@ -50,7 +50,7 @@ import Control.Exception
 import GHC.Paths
 #endif
 
-import Debug.Trace
+--import Debug.Trace
 
 data DatabaseContextExprDetails = CountUpdatedTuples
 
@@ -282,7 +282,7 @@ evalGraphRefDatabaseContextExpr (Assign relVarName expr) = do
   context <- getStateContext
   let existingRelVar = M.lookup relVarName (relationVariables context)
       reEnv = freshGraphRefRelationalExprEnv (Just context) graph
-      eNewExprType = runGraphRefRelationalExprM reEnv (typeForGraphRefRelationalExpr expr)      
+
   case existingRelVar of
     Nothing -> do
       case runGraphRefRelationalExprM reEnv (typeForGraphRefRelationalExpr expr) of
@@ -294,13 +294,16 @@ evalGraphRefDatabaseContextExpr (Assign relVarName expr) = do
       let eExpectedType = runGraphRefRelationalExprM reEnv (typeForGraphRefRelationalExpr existingRel)
       case eExpectedType of
         Left err -> dbErr err
-        Right expectedType ->
+        Right expectedType -> do
+      -- if we are targeting an existing rv, we can morph a MakeRelationFromExprs datum to fill in missing type variables'
+          let hintedExpr = addTargetTypeHints (attributes expectedType) expr
+              eNewExprType = runGraphRefRelationalExprM reEnv (typeForGraphRefRelationalExpr hintedExpr)
           case eNewExprType of
             Left err -> dbErr err
             Right newExprType -> do
               if newExprType == expectedType then do
                 lift $ except $ validateAttributes (typeConstructorMapping context) (attributes newExprType)
-                setRelVar relVarName expr 
+                setRelVar relVarName hintedExpr 
               else
                 dbErr (RelationTypeMismatchError (attributes expectedType) (attributes newExprType))
 
@@ -896,7 +899,7 @@ evalGraphRefAtomExpr tupIn (IfThenAtomExpr ifExpr thenExpr elseExpr) = do
   case conditional of
     BoolAtom True -> evalGraphRefAtomExpr tupIn thenExpr
     BoolAtom False -> evalGraphRefAtomExpr tupIn elseExpr
-    otherAtom -> traceShow ("evalAtom"::String, otherAtom) $ throwError (IfThenExprExpectedBooleanError (atomTypeForAtom otherAtom))
+    otherAtom -> throwError (IfThenExprExpectedBooleanError (atomTypeForAtom otherAtom))
 evalGraphRefAtomExpr _ (ConstructedAtomExpr tOrF [] _)
   | tOrF == "True" = pure (BoolAtom True)
   | tOrF == "False" = pure (BoolAtom False)
@@ -1513,4 +1516,38 @@ firstAtomForAttributeName attrName tuples = do
   case foldr folder Nothing tuples of
     Nothing -> throwError (NoSuchAttributeNamesError (S.singleton attrName))
     Just match -> pure match
-
+
+-- | Optionally add type hints to resolve type variables. For example, if we are inserting into a known relvar, then we have its concrete type.    
+addTargetTypeHints :: Attributes -> GraphRefRelationalExpr -> GraphRefRelationalExpr
+addTargetTypeHints targetAttrs expr =
+  case expr of
+    MakeRelationFromExprs Nothing tupExprs ->
+      MakeRelationFromExprs (Just targetAttrExprs) tupExprs
+    Project attrs e ->
+      Project attrs (hint e)
+    Union a b ->
+      Union (hint a) (hint b)
+    Join a b ->
+      Join (hint a) (hint b)
+    Rename rens e ->
+      Rename rens (hint e)
+    Difference a b ->
+      Difference (hint a) (hint b)
+    Group attrs gname e ->
+      Group attrs gname (hint e)
+    Ungroup gname e ->
+      Ungroup gname (hint e)
+    Restrict restriction e ->
+      Restrict restriction (hint e)
+    Equals a b ->
+      Equals (hint a) (hint b)
+    NotEquals a b ->
+      NotEquals (hint a) (hint b)
+    Extend tupExprs e ->
+      Extend tupExprs (hint e)
+    With withs e ->
+      With withs (hint e)
+    _ -> expr
+  where
+    targetAttrExprs = map NakedAttributeExpr (A.toList targetAttrs)
+    hint = addTargetTypeHints targetAttrs

src/lib/ProjectM36/SQL/Convert.hs

@@ -16,7 +16,7 @@ import ProjectM36.SQL.DropTable as DropTable
 import ProjectM36.RelationalExpression
 import ProjectM36.DataFrame (DataFrameExpr(..), AttributeOrderExpr(..), Order(..), usesDataFrameFeatures)
 import ProjectM36.AttributeNames as A
-import ProjectM36.Relation (attributes)
+import ProjectM36.Relation (attributes, atomTypeForName)
 import qualified ProjectM36.Attribute as A
 import qualified Data.Text as T
 import qualified ProjectM36.WithNameExpr as With
@@ -36,7 +36,7 @@ import Data.Foldable (foldl')
 import Data.Bifunctor (bimap)
 --import qualified Data.HashSet as HS
 
---import Debug.Trace
+import Debug.Trace
 
 {-
 TODO
@@ -478,7 +478,7 @@ convertSelect typeF sel = do
       finalRelExpr = explicitWithF (withF (projF (convertExpr dfExpr)))
   -- if we have only one table alias or the columns are all unambiguous, remove table aliasing of attributes
   -- apply rename reduction- this could be applied by the static query optimizer, but we do it here to simplify the tests so that they aren't polluted with redundant renames
---  traceShowM ("final expr"::String, finalRelExpr)
+  traceShowM ("final expr"::String, finalRelExpr)
   pure (dfExpr { convertExpr = finalRelExpr })
 
 
@@ -732,7 +732,7 @@ convertScalarExpr typeF expr = do
       BooleanLiteral False -> naked (BoolAtom False)
       --pure $ ConstructedAtomExpr "False" [] ()
       -- we don't have enough type context with a cast, so we default to text
-      NullLiteral -> pure $ ConstructedAtomExpr "SQLNull" [] ()
+      NullLiteral -> pure $ ConstructedAtomExpr "SQLNullOfUnknownType" [] ()
       Identifier i -> do
         AttributeAtomExpr <$> convertColumnName i
       BinaryOperator exprA op exprB -> do
@@ -760,7 +760,7 @@ convertProjectionScalarExpr typeF expr = do
       BooleanLiteral False ->
         naked (BoolAtom False)
         --pure $ ConstructedAtomExpr "False" [] ()
-      NullLiteral -> pure $ ConstructedAtomExpr "SQLNull" [] ()
+      NullLiteral -> pure $ ConstructedAtomExpr "SQLNullOfUnknownType" [] ()
       Identifier i -> do
         AttributeAtomExpr <$> convertColumnProjectionName i
       BinaryOperator exprA op exprB -> do
@@ -1165,27 +1165,102 @@ convertDBUpdate typeF (UpdateDropTable dt) = convertDropTable typeF dt
 
 convertInsert :: TypeForRelExprF -> Insert -> ConvertM DatabaseContextExpr
 convertInsert typeF ins = do
-  dfExpr <- convertQuery typeF (source ins)
-  when (usesDataFrameFeatures dfExpr) $ throwSQLE (NotSupportedError "ORDER BY/LIMIT/OFFSET in subquery")
   -- check that all columns are mentioned because Project:M36 does not support default columns
-  case typeF (convertExpr dfExpr) of
+  rvTarget <- convertTableName (Insert.target ins)
+  let eRvTargetType = typeF (RelationVariable rvTarget ())
+  case eRvTargetType of
     Left err -> throwSQLE (SQLRelationalError err)
-    Right rvExprType -> do
-      let rvExprAttrNames = A.attributeNamesList (attributes rvExprType)
-          insAttrNames = map convertUnqualifiedColumnName (Insert.targetColumns ins)
-          rvExprColNameSet = S.map UnqualifiedColumnName (S.fromList rvExprAttrNames)
-          insAttrColSet = S.fromList (Insert.targetColumns ins)
-      when (length rvExprAttrNames /= length insAttrNames) $ throwSQLE (ColumnNamesMismatch rvExprColNameSet insAttrColSet)
-      rvTarget <- convertTableName (Insert.target ins)
-      -- insert into s(s#,sname,city,status) select * from s; -- we need to reorder attributes to align?
-      -- rename attributes rexpr via query/values to map to targetCol attrs
-      let insExpr = if rvExprColNameSet == insAttrColSet then -- if the attributes already align, don't perform any renaming
-                      convertExpr dfExpr
-                    else
-                      Rename (S.fromList (filter rendundantRename (zip rvExprAttrNames insAttrNames))) (convertExpr dfExpr)
-          rendundantRename (a,b) = a /= b
-      --traceShowM ("ins"::String, insExpr)
-      pure $ B.Insert rvTarget insExpr
+    Right rvTargetType -> do
+      -- if types do not align due to nullability, then add SQLJust
+      dfExpr <- convertQuery typeF (source ins)
+      when (usesDataFrameFeatures dfExpr) $ throwSQLE (NotSupportedError "ORDER BY/LIMIT/OFFSET in subquery")
+--      traceShowM ("before dfExpr"::String, dfExpr)
+      case typeF (convertExpr dfExpr) of
+        Left err -> throwSQLE (SQLRelationalError err)
+        Right rvExprType -> do
+--          traceShowM ("after dfExpr"::String, rvExprType)          
+          let rvExprAttrNames = A.attributeNamesList (attributes rvExprType)
+              insAttrNames = map convertUnqualifiedColumnName (Insert.targetColumns ins)
+              rvExprColNameSet = S.map UnqualifiedColumnName (S.fromList rvExprAttrNames)
+              insAttrColSet = S.fromList (Insert.targetColumns ins)
+          when (length rvExprAttrNames /= length insAttrNames) $ throwSQLE (ColumnNamesMismatch rvExprColNameSet insAttrColSet)
+
+
+          -- insert into s(s#,sname,city,status) select * from s; -- we need to reorder attributes to align?
+          -- rename attributes rexpr via query/values to map to targetCol attrs
+          let atomTypeForName' attrName type' =
+                case atomTypeForName attrName type' of
+                  Left err -> throwSQLE (SQLRelationalError err)
+                  Right targetType -> pure targetType
+              ren a b (Rename names expr) = Rename (S.insert (a,b) names) expr
+              ren a b e = Rename (S.singleton (a, b)) e
+              sqlPrefix s = "_sql_" <> s
+              projHide n = Project (InvertedAttributeNames (S.singleton n))
+              -- if one of the types is a nullable version of the other
+--              isSQLNullableCombo t1 t2 = isSQLNullableSpecificType t1 t2 || isSQLNullableSpecificType t2 t1
+              sqlNullMorpher interName targetName targetType t2 expr
+                | isSQLNullableSpecificType targetType t2 = -- targetType is nullable version of t2
+                  Extend (AttributeExtendTupleExpr targetName (ConstructedAtomExpr "SQLJust" [AttributeAtomExpr interName] ())) expr
+                | otherwise = expr
+
+          let typeMatchRenamer acc (targetAttrName, sourceAttrName) = do
+                targetType <- atomTypeForName' targetAttrName rvTargetType
+                insType <- atomTypeForName' sourceAttrName rvExprType
+                if targetType == insType && targetAttrName == sourceAttrName then --nothing to do
+                  pure acc
+                  else if targetAttrName /= sourceAttrName &&
+                          targetType == insType then do
+                          --simple rename
+--                         traceShowM ("simple rename"::String)
+                         pure $ ren sourceAttrName targetAttrName acc
+                  else if targetAttrName == sourceAttrName &&
+                          targetType /= insType &&
+                          isSQLNullableSpecificType targetType insType
+                  then do -- we need to extend the expr, but we want to use the targetName, so we have to rename it twice
+--                         traceShowM ("same name, null conversion"::String)
+                         let intermediateName = sqlPrefix targetAttrName
+                         pure $ ren intermediateName targetAttrName (sqlNullMorpher intermediateName targetAttrName targetType insType (ren sourceAttrName intermediateName acc))
+                  else if targetAttrName /= sourceAttrName &&
+                          targetType /= insType &&
+                          isSQLNullableSpecificType targetType insType then do
+                         -- we extend the expr, but don't need an intermediate rename
+--                         traceShowM ("diff name, null conversion"::String)
+                         pure $ projHide sourceAttrName (Extend (AttributeExtendTupleExpr targetAttrName (ConstructedAtomExpr "SQLJust" [AttributeAtomExpr sourceAttrName] ())) acc)
+                  else if targetAttrName == sourceAttrName &&
+                          isSQLNullUnknownType insType &&
+                          isNullAtomType targetType then do
+--                         traceShowM ("same name, unknown null"::String)
+                         case atomTypeFromSQLNull targetType of
+                           Nothing -> do
+                             pure acc
+                           -- replace null of unknown type with typed null
+                           Just atype -> do
+                               pure $ Extend (AttributeExtendTupleExpr targetAttrName (NakedAtomExpr (nullAtom atype Nothing))) (projHide sourceAttrName acc)
+                  else if targetAttrName /= sourceAttrName &&
+                          isSQLNullUnknownType insType &&
+                          isNullAtomType targetType then do
+--                         traceShowM ("different name, unknown null"::String, targetAttrName, sourceAttrName, targetType)
+                         case atomTypeFromSQLNull targetType of
+                           Nothing -> do
+                             pure acc
+                           -- replace null of unknown type with typed null
+                           Just _atype -> do
+                               pure $ projHide sourceAttrName $ Extend (AttributeExtendTupleExpr targetAttrName (ConstructedAtomExpr "SQLNull" [] ())) acc
+                  else
+                         pure acc
+
+          insExpr <- foldM typeMatchRenamer (convertExpr dfExpr) (zip insAttrNames rvExprAttrNames)
+{-          let insExpr = if rvExprColNameSet == insAttrColSet then -- if the attributes already align, don't perform any renaming
+                          convertExpr dfExpr
+                        else
+                          Rename (S.fromList (filter rendundantRename (zip rvExprAttrNames insAttrNames))) (convertExpr dfExpr)
+              rendundantRename (a,b) = a /= b-}
+{-          traceShowM ("source ins"::String, source ins)
+          traceShowM ("source ins converted"::String, convertExpr dfExpr)
+          traceShowM ("ins converted"::String, insExpr)
+          traceShowM ("rvTargetType"::String, rvTargetType)-}
+
+          pure $ B.Insert rvTarget insExpr
 
 convertDelete :: TypeForRelExprF -> Delete.Delete -> ConvertM DatabaseContextExpr
 convertDelete typeF del = do