WIP SQL NULL tests passing

project-m36.cabal

@@ -107,6 +107,8 @@ Library
                      ProjectM36.DataTypes.Primitive,
                      ProjectM36.DataTypes.Interval,
                      ProjectM36.DataTypes.ByteString,
+                     ProjectM36.DataTypes.SQL.Null,
+                     ProjectM36.SQLDatabaseContext,
                      ProjectM36.MiscUtils,
                      ProjectM36.Notifications,
                      ProjectM36.Relation,

src/bin/SQL/Interpreter/Base.hs

@@ -23,8 +23,12 @@ reserved word = do
 reserveds :: Text -> Parser ()
 reserveds words' = do
   let words'' = T.splitOn " " words'
-  sequence_ (map reserved words'')
+  reserveds' words''
 
+reserveds' :: [Text] -> Parser ()
+reserveds' words' =
+  sequence_ (map reserved words')
+
 -- does not consume trailing spaces
 qualifiedNameSegment :: Text -> Parser Text
 qualifiedNameSegment sym = T.toLower <$> string' sym

src/bin/SQL/Interpreter/Convert.hs

@@ -17,7 +17,6 @@ import Data.List (intercalate, find)
 import qualified Data.Functor.Foldable as Fold
 import qualified Data.List.NonEmpty as NE
 import Control.Monad (when)
-import ProjectM36.DataTypes.Maybe
 import Data.Maybe (isJust, fromMaybe)
 --import Control.Monad (void)
 import Control.Monad.Trans.State (StateT, get, put, runStateT, evalStateT)
@@ -737,6 +736,12 @@ convertWhereClause typeF (RestrictionExpr rexpr) = do
         b <- convertScalarExpr typeF exprB
         f <- lookupOperator op
         pure (AtomExprPredicate (f [a,b]))
+      PostfixOperator expr (OperatorName ops) -> do
+        expr' <- convertScalarExpr typeF expr
+--        traceShowM ("convertWhereClause"::String, expr')
+        case ops of
+          ["is", "null"] -> do
+            pure $ AtomExprPredicate (FunctionAtomExpr "sql_isnull" [expr'] ())
       InExpr inOrNotIn sexpr (InList matches') -> do
         eqExpr <- convertScalarExpr typeF sexpr
         let (match:matches) = reverse matches'
@@ -764,8 +769,10 @@ convertScalarExpr typeF expr = do
       IntegerLiteral i -> naked (IntegerAtom i)
       DoubleLiteral d -> naked (DoubleAtom d)
       StringLiteral s -> naked (TextAtom s)
+      BooleanLiteral True -> pure $ ConstructedAtomExpr "True" [] ()
+      BooleanLiteral False -> pure $ ConstructedAtomExpr "False" [] ()
       -- we don't have enough type context with a cast, so we default to text
-      NullLiteral -> naked (ConstructedAtom "Nothing" (maybeAtomType TextAtomType) [])
+      NullLiteral -> pure $ ConstructedAtomExpr "SQLNull" [] ()
       Identifier i -> do
         AttributeAtomExpr <$> convertColumnName i
       BinaryOperator exprA op exprB -> do
@@ -781,7 +788,9 @@ convertProjectionScalarExpr typeF expr = do
       IntegerLiteral i -> naked (IntegerAtom i)
       DoubleLiteral d -> naked (DoubleAtom d)
       StringLiteral s -> naked (TextAtom s)
-      NullLiteral -> naked (ConstructedAtom "Nothing" (maybeAtomType TextAtomType) [])      
+      BooleanLiteral True -> pure $ ConstructedAtomExpr "True" [] ()
+      BooleanLiteral False -> pure $ ConstructedAtomExpr "False" [] ()
+      NullLiteral -> pure $ ConstructedAtomExpr "SQLNull" [] ()
       Identifier i ->
         AttributeAtomExpr <$> convertColumnProjectionName i
       BinaryOperator exprA op exprB -> do
@@ -982,11 +991,11 @@ lookupFunc qname =
                  ("<",f "lt"),
                  (">=",f "gte"),
                  ("<=",f "lte"),
-                 ("=",f "eq"),
+                 ("=",f "sql_eq"),
                  ("!=",f "not_eq"), -- function missing
                  ("<>",f "not_eq"), -- function missing
                  ("+", f "add"),
-                 ("and", f "and"),
+                 ("and", f "sql_and"),
                  ("or", f "or")
                ]
 

src/bin/SQL/Interpreter/Select.hs

@@ -63,12 +63,13 @@ data ScalarExprBase n =
   IntegerLiteral Integer
   | DoubleLiteral Double
   | StringLiteral Text
+  | BooleanLiteral Bool
   | NullLiteral
     -- | Interval
   | Identifier n
   | BinaryOperator (ScalarExprBase n) OperatorName (ScalarExprBase n)
   | PrefixOperator OperatorName (ScalarExprBase n)
-  | PostfixOperator (ScalarExprBase n) ColumnName
+  | PostfixOperator (ScalarExprBase n) OperatorName
   | BetweenOperator (ScalarExprBase n) (ScalarExprBase n) (ScalarExprBase n)
   | FunctionApplication FuncName (ScalarExprBase n)
   | CaseExpr { caseWhens :: [([ScalarExprBase n],ScalarExprBase n)],
@@ -297,6 +298,7 @@ scalarExprOp =
      --binarySymbolsN ["not", "like"]
      ],
     map binarySymbolN ["<",">",">=","<=","!=","<>","="],
+    [postfixKeywords ["is","null"]],
 {-    [binarySymbolsN ["is", "distinct", "from"],
      binarySymbolsN ["is", "not", "distinct", "from"]],-}
     [binarySymbolL "and"],
@@ -313,6 +315,10 @@ scalarExprOp =
    binarySymbolR s = E.InfixR $ binary s
    binarySymbolN s = E.InfixN $ binary s
    qComparisonOp = E.Postfix $ try quantifiedComparisonSuffixP
+   postfixKeywords kws = E.Postfix $ do
+     try $ reserveds' kws
+     pure (\a -> PostfixOperator a (OperatorName kws))
+
 
 qualifiedOperatorP :: Text -> Parser OperatorName
 qualifiedOperatorP sym =
@@ -379,11 +385,14 @@ comparisonOperatorP = choice (map (\(match', op) -> reserved match' $> op) ops)
                  ("!=", OpNE)]
 
 simpleLiteralP :: Parser (ScalarExprBase a)
-simpleLiteralP = try doubleLiteralP <|> integerLiteralP <|> stringLiteralP <|> nullLiteralP
+simpleLiteralP = try doubleLiteralP <|> integerLiteralP <|> booleanLiteralP <|> stringLiteralP <|> nullLiteralP
 
 doubleLiteralP :: Parser (ScalarExprBase a)
 doubleLiteralP = DoubleLiteral <$> double
 
+booleanLiteralP :: Parser (ScalarExprBase a)
+booleanLiteralP = BooleanLiteral <$> ((reserved "true" $> True) <|> (reserved "false" $> False))
+
 integerLiteralP :: Parser (ScalarExprBase a)
 integerLiteralP = IntegerLiteral <$> integer
 

src/bin/TutorialD/tutd.hs

@@ -4,6 +4,8 @@ import ProjectM36.Base
 import ProjectM36.Client
 import ProjectM36.Server.ParseArgs
 import ProjectM36.Server
+import ProjectM36.DatabaseContext
+import ProjectM36.SQLDatabaseContext
 import System.IO
 import GHC.IO.Encoding
 import Options.Applicative
@@ -44,6 +46,8 @@ opts = info (parseArgs <**> helpOption) idm
 
 connectionInfoForConfig :: InterpreterConfig -> ConnectionInfo
 connectionInfoForConfig (LocalInterpreterConfig pStrategy _ _ ghcPkgPaths _) = InProcessConnectionInfo pStrategy outputNotificationCallback ghcPkgPaths
+  --basicDatabaseContext
+  sqlDatabaseContext -- for testing sql functions ONLY! DO NOT COMMIT
 connectionInfoForConfig (RemoteInterpreterConfig remoteHost remotePort remoteDBName _ _ _) = RemoteConnectionInfo remoteDBName remoteHost (show remotePort) outputNotificationCallback
 
 headNameForConfig :: InterpreterConfig -> HeadName

src/lib/ProjectM36/Client.hs

@@ -120,7 +120,6 @@ import ProjectM36.DatabaseContextFunction as DCF
 import qualified ProjectM36.IsomorphicSchema as Schema
 import Control.Monad.State
 import qualified ProjectM36.RelationalExpression as RE
-import ProjectM36.DatabaseContext (basicDatabaseContext)
 import qualified ProjectM36.TransactionGraph as Graph
 import ProjectM36.TransactionGraph as TG
 import qualified ProjectM36.Transaction as Trans
@@ -193,7 +192,7 @@ data RequestTimeoutException = RequestTimeoutException
 instance Exception RequestTimeoutException
 
 -- | Construct a 'ConnectionInfo' to describe how to make the 'Connection'. The database can be run within the current process or running remotely via RPC.
-data ConnectionInfo = InProcessConnectionInfo PersistenceStrategy NotificationCallback [GhcPkgPath] |
+data ConnectionInfo = InProcessConnectionInfo PersistenceStrategy NotificationCallback [GhcPkgPath] DatabaseContext |
                       RemoteConnectionInfo DatabaseName Hostname ServiceName NotificationCallback
 
 type EvaluatedNotifications = M.Map NotificationName EvaluatedNotification
@@ -260,11 +259,10 @@ createScriptSession ghcPkgPaths = do
 -- | To create a 'Connection' to a remote or local database, create a 'ConnectionInfo' and call 'connectProjectM36'.
 connectProjectM36 :: ConnectionInfo -> IO (Either ConnectionError Connection)
 --create a new in-memory database/transaction graph
-connectProjectM36 (InProcessConnectionInfo strat notificationCallback ghcPkgPaths) = do
+connectProjectM36 (InProcessConnectionInfo strat notificationCallback ghcPkgPaths bootstrapDatabaseContext) = do
   freshId <- nextRandom
   tstamp <- getCurrentTime
-  let bootstrapContext = basicDatabaseContext 
-      freshGraph = bootstrapTransactionGraph tstamp freshId bootstrapContext
+  let freshGraph = bootstrapTransactionGraph tstamp freshId bootstrapDatabaseContext
   case strat of
     --create date examples graph for now- probably should be empty context in the future
     NoPersistence -> do

src/lib/ProjectM36/DataTypes/SQL/Null.hs

@@ -0,0 +1,83 @@
+module ProjectM36.DataTypes.SQL.Null where
+import ProjectM36.Base
+import ProjectM36.AtomFunctionError
+import qualified Data.Map as M
+import qualified Data.HashSet as HS
+
+-- analogous but not equivalent to a Maybe type due to how NULLs interact with every other value
+
+nullAtomType :: AtomType -> AtomType
+nullAtomType arg = ConstructedAtomType "SQLNullable" (M.singleton "a" arg)
+
+nullTypeConstructorMapping :: TypeConstructorMapping
+nullTypeConstructorMapping = [(ADTypeConstructorDef "SQLNullable" ["a"],
+                                [DataConstructorDef "SQLNull" [],
+                                 DataConstructorDef "SQLJust" [DataConstructorDefTypeVarNameArg "a"]])
+                              ]
+
+nullAtomFunctions :: AtomFunctions
+nullAtomFunctions = HS.fromList [
+  Function {
+      funcName = "sql_isnull", --this function works on any type variable, not just SQLNullable because removing the isnull function in cases where the type is clearly not SQLNullable is more difficult
+      funcType = [TypeVariableType "a", BoolAtomType],
+      funcBody = FunctionBuiltInBody $
+        \case
+           a:[] -> pure $ BoolAtom (isNull a)
+           _ -> Left AtomFunctionTypeMismatchError
+      },
+    Function {
+      funcName = "sql_equals",
+      funcType = [nullAtomType (TypeVariableType "a"),
+                  nullAtomType (TypeVariableType "a"),
+                  nullAtomType BoolAtomType],
+      funcBody = FunctionBuiltInBody nullEq
+      },
+    Function {
+      funcName = "sql_and",
+      funcType = [TypeVariableType "a", TypeVariableType "b", BoolAtomType], -- for a more advanced typechecker, this should be BoolAtomType or SQLNullable BoolAtomType
+      funcBody = FunctionBuiltInBody nullAnd
+      }
+    ]
+  where
+    sqlNull typ = ConstructedAtom "SQLNull" typ []
+    sqlNullable val typ = ConstructedAtom "SQLJust" typ [val]
+    isNull (ConstructedAtom dConsName _ _) | dConsName == "SQLNull" = True
+    isNull _ = False
+    nullEq :: AtomFunctionBodyType
+    nullEq (a@(ConstructedAtom _ typA argsA) : b@(ConstructedAtom _ _ argsB) : [])
+      | isNull a || isNull b = pure $ sqlNull typA
+      | otherwise = pure $ sqlNullable (BoolAtom $ argsA == argsB) BoolAtomType
+    nullEq _ = Left AtomFunctionTypeMismatchError
+
+isSQLBool :: Atom -> Bool
+isSQLBool (ConstructedAtom dConsName BoolAtomType [_]) | dConsName == "SQLNullable" = True
+isSQLBool (BoolAtom _) = True
+isSQLBool _ = False
+
+sqlBool :: Atom -> Maybe Bool
+sqlBool (ConstructedAtom dConsName BoolAtomType [BoolAtom tf]) | dConsName == "SQLJust" = Just tf
+sqlBool (ConstructedAtom dConsName BoolAtomType []) | dConsName == "SQLNull" = Nothing
+sqlBool (BoolAtom tf) = Just tf
+sqlBool x | isSQLBool x = error "internal sqlBool type error" -- should be caught above
+sqlBool _ = error "sqlBool type mismatch"
+
+
+nullAnd :: [Atom] -> Either AtomFunctionError Atom
+nullAnd [a,b] | isSQLBool a && isSQLBool b = do
+                  let bNull = nullAtom BoolAtomType Nothing
+                      boolF = nullAtom BoolAtomType (Just (BoolAtom False))
+                  pure $ case (sqlBool a, sqlBool b) of
+                             (Nothing, Nothing) -> bNull
+                             (Nothing, Just True) -> bNull
+                             (Nothing, Just False) -> boolF
+                             (Just True, Nothing) -> bNull
+                             (Just False, Nothing) -> boolF
+                             (Just a', Just b') ->
+                               nullAtom BoolAtomType (Just (BoolAtom (a' && b')))
+nullAnd _ = Left AtomFunctionTypeMismatchError
+
+nullAtom :: AtomType -> Maybe Atom -> Atom
+nullAtom aType mAtom =
+  case mAtom of
+    Nothing -> ConstructedAtom "SQLNull" (nullAtomType aType) []
+    Just atom -> ConstructedAtom "SQLJust" (nullAtomType aType) [atom]

src/lib/ProjectM36/SQLDatabaseContext.hs

@@ -0,0 +1,13 @@
+-- | Enables SQL-equivalent features such as NULL types in the database in addition to Project:M36 basic functions.
+module ProjectM36.SQLDatabaseContext where
+import ProjectM36.Base
+import ProjectM36.DatabaseContext
+import ProjectM36.DataTypes.SQL.Null
+
+sqlDatabaseContext :: DatabaseContext
+sqlDatabaseContext = basicDatabaseContext { atomFunctions =
+                                            atomFunctions basicDatabaseContext <> nullAtomFunctions,
+                                            typeConstructorMapping =
+                                            typeConstructorMapping basicDatabaseContext <> nullTypeConstructorMapping
+                                            }
+

src/lib/ProjectM36/Server.hs

@@ -6,6 +6,7 @@ import ProjectM36.Server.EntryPoints
 import ProjectM36.Server.RemoteCallTypes
 import ProjectM36.Server.Config (ServerConfig(..))
 import ProjectM36.FSType
+import ProjectM36.DatabaseContext
 
 import Control.Concurrent.MVar (MVar)
 import System.IO (stderr, hPutStrLn)
@@ -199,7 +200,7 @@ launchServer daemonConfig mAddr = do
     hPutStrLn stderr checkFSErrorMsg
     pure False
     else do
-      econn <- connectProjectM36 (InProcessConnectionInfo (persistenceStrategy daemonConfig) loggingNotificationCallback (ghcPkgPaths daemonConfig))
+      econn <- connectProjectM36 (InProcessConnectionInfo (persistenceStrategy daemonConfig) loggingNotificationCallback (ghcPkgPaths daemonConfig) basicDatabaseContext)
       case econn of 
         Left err -> do      
           hPutStrLn stderr ("Failed to create database connection: " ++ show err)