refactor Select+Query types

add multi-SQL-expression DBUpdate function via Client for batch jobs
basic INSERT, UPDATE, DELETE support functioning

project-m36.cabal

@@ -139,7 +139,10 @@ Library
                      ProjectM36.RegisteredQuery,
                      ProjectM36.SQL.Convert,
                      ProjectM36.SQL.Select,
-                     ProjectM36.SQL.Update
+                     ProjectM36.SQL.Update,
+                     ProjectM36.SQL.Insert,
+                     ProjectM36.SQL.Delete,
+                     ProjectM36.SQL.DBUpdate
     GHC-Options: -Wall -rdynamic
     if os(windows)
       Build-Depends: Win32 >= 2.12
@@ -287,7 +290,10 @@ Executable sqlegacy
                    SQL.Interpreter,
                    SQL.Interpreter.TransactionGraphOperator,
                    SQL.Interpreter.ImportBasicExample,
-                   SQL.Interpreter.Update
+                   SQL.Interpreter.Update,
+                   SQL.Interpreter.Insert,
+                   SQL.Interpreter.Delete,                   
+                   SQL.Interpreter.DBUpdate
 
     Main-Is: ./SQL/Interpreter/sqlegacy.hs
     if os(windows)

src/bin/SQL/Interpreter.hs

@@ -2,36 +2,34 @@ module SQL.Interpreter where
 import ProjectM36.Base
 import ProjectM36.Interpreter
 import ProjectM36.SQL.Select
-import ProjectM36.SQL.Update
 import ProjectM36.DatabaseContext
 import ProjectM36.DateExamples
 import ProjectM36.Error
 import SQL.Interpreter.ImportBasicExample
 import SQL.Interpreter.TransactionGraphOperator
 import SQL.Interpreter.Select
-import SQL.Interpreter.Update
+import SQL.Interpreter.DBUpdate
+import ProjectM36.SQL.DBUpdate
 import qualified Data.Text as T
 import qualified ProjectM36.Client as C
 import Text.Megaparsec
 import SQL.Interpreter.Base
 
-data SQLCommand = RODatabaseContextOp Select | -- SELECT
+data SQLCommand = RODatabaseContextOp Query | -- SELECT
                   DatabaseContextExprOp DatabaseContextExpr |
-                  UpdateOp Update | -- UPDATE, DELETE, INSERT
---                  InsertOp Insert |
---                  DeleteOp Delete |
+                  DBUpdateOp [DBUpdate] | -- INSERT, UPDATE, DELETE
                   ImportBasicExampleOp ImportBasicExampleOperator |  -- IMPORT EXAMPLE cjdate
                   TransactionGraphOp TransactionGraphOperator -- COMMIT, ROLLBACK
                 deriving (Show)
 
 parseSQLUserInput :: T.Text -> Either ParserError SQLCommand
-parseSQLUserInput = parse ((parseRODatabaseContextOp <|>
+parseSQLUserInput = parse ((parseRODatabaseContextOp <* semi) <|>
                            parseDatabaseContextExprOp <|>
-                           parseTransactionGraphOp <|>
-                           parseImportBasicExampleOp) <* semi) ""
+                           (parseTransactionGraphOp <* semi) <|>
+                           (parseImportBasicExampleOp <* semi)) ""
 
 parseRODatabaseContextOp :: Parser SQLCommand
-parseRODatabaseContextOp = RODatabaseContextOp <$> queryExprP
+parseRODatabaseContextOp = RODatabaseContextOp <$> queryP
 
 parseImportBasicExampleOp :: Parser SQLCommand
 parseImportBasicExampleOp = ImportBasicExampleOp <$> importBasicExampleP
@@ -40,14 +38,14 @@ parseTransactionGraphOp :: Parser SQLCommand
 parseTransactionGraphOp = TransactionGraphOp <$> transactionGraphOperatorP
 
 parseDatabaseContextExprOp :: Parser SQLCommand
-parseDatabaseContextExprOp = UpdateOp <$> updateP  -- <|> insertP)
+parseDatabaseContextExprOp = DBUpdateOp <$> dbUpdatesP
 
 evalSQLInteractive :: C.SessionId -> C.Connection -> SafeEvaluationFlag -> InteractiveConsole -> SQLCommand -> IO ConsoleResult
 evalSQLInteractive sessionId conn safeFlag interactiveConsole command =
   case command of
-    RODatabaseContextOp sel -> do
+    RODatabaseContextOp query -> do
       --get relvars to build conversion context
-      eDFExpr <- C.convertSQLSelect sessionId conn sel
+      eDFExpr <- C.convertSQLQuery sessionId conn query
       case eDFExpr of
         Left err -> pure $ DisplayRelationalErrorResult err
         Right dfExpr -> do
@@ -62,8 +60,8 @@ evalSQLInteractive sessionId conn safeFlag interactiveConsole command =
           pure (DisplayErrorResult ("No such example: " <> exampleName))
     DatabaseContextExprOp dbcExpr -> do
         eHandler $ C.executeDatabaseContextExpr sessionId conn dbcExpr
-    UpdateOp up -> do
-      eDBCExpr <- C.convertSQLUpdate sessionId conn up
+    DBUpdateOp updates -> do
+      eDBCExpr <- C.convertSQLDBUpdates sessionId conn updates
       case eDBCExpr of
         Left err -> pure $ DisplayRelationalErrorResult err
         Right dbcExpr -> 

src/bin/SQL/Interpreter/DBUpdate.hs

@@ -0,0 +1,17 @@
+module SQL.Interpreter.DBUpdate where
+import ProjectM36.Interpreter
+import ProjectM36.SQL.DBUpdate
+import SQL.Interpreter.Update
+import SQL.Interpreter.Insert
+import SQL.Interpreter.Delete
+import SQL.Interpreter.Base
+import Text.Megaparsec
+
+dbUpdatesP :: Parser [DBUpdate]
+dbUpdatesP = some dbUpdateP
+
+dbUpdateP :: Parser DBUpdate
+dbUpdateP = (UpdateUpdate <$> updateP <* semi) <|>
+            (UpdateInsert <$> insertP <* semi) <|>
+            (UpdateDelete <$> deleteP <* semi)
+

src/bin/SQL/Interpreter/Delete.hs

@@ -0,0 +1,15 @@
+module SQL.Interpreter.Delete where
+import SQL.Interpreter.Select
+import ProjectM36.SQL.Delete
+import ProjectM36.SQL.Select
+import SQL.Interpreter.Base
+import ProjectM36.Interpreter
+import Control.Applicative
+
+deleteP :: Parser Delete
+deleteP = do
+  reserveds "delete from"
+  tname <- tableNameP
+  restrictExpr <- whereP <|> pure (RestrictionExpr (BooleanLiteral True))
+  pure $ Delete { target = tname,
+                  restriction = restrictExpr }

src/bin/SQL/Interpreter/Insert.hs

@@ -0,0 +1,15 @@
+module SQL.Interpreter.Insert where
+import SQL.Interpreter.Select
+import ProjectM36.SQL.Insert
+import SQL.Interpreter.Base
+import ProjectM36.Interpreter
+
+insertP :: Parser Insert
+insertP = do
+  reserveds "insert into"
+  tname <- tableNameP
+  colNames <- parens (sepByComma1 unqualifiedColumnNameP)
+  q <- queryP
+  pure (Insert { target = tname,
+                 targetColumns = colNames,
+                 source = q })

src/bin/SQL/Interpreter/Select.hs

@@ -13,16 +13,28 @@ import qualified Data.List.NonEmpty as NE
 
 
 parseSelect :: Text -> Either ParserError Select
-parseSelect = parse (queryExprP <* semi <* eof) "<interactive>"
+parseSelect = parse (selectP <* semi <* eof) "<interactive>"
+
+parseQuery :: Text -> Either ParserError Query
+parseQuery = parse (queryP <* semi <* eof) "<interactive>"
 
-queryExprP :: Parser Select
-queryExprP = tableP <|> selectP
+queryP :: Parser Query
+queryP = (QuerySelect <$> selectP) <|>
+         (QueryValues <$> valuesP) <|>
+         (QueryTable <$> tableP)
+
+valuesP :: Parser [[ScalarExpr]]
+valuesP = do
+  reserved "values"
+  sepByComma1 tupleP
+
+tupleP :: Parser [ScalarExpr]
+tupleP = parens (sepByComma1 scalarExprP)
 
-tableP :: Parser Select
+tableP :: Parser TableName
 tableP = do
   reserved "table"
-  tname <- tableNameP
-  pure $ emptySelect { tableExpr = Just $ emptyTableExpr { fromClause = [SimpleTableRef tname] } }
+  tableNameP
 
 tableNameP :: Parser TableName
 tableNameP = TableName <$> qualifiedNameP'

src/lib/ProjectM36/Client.hs

@@ -48,8 +48,8 @@ module ProjectM36.Client
        defaultHeadName,
        addClientNode,
        getDDLHash,
-       convertSQLSelect,
-       convertSQLUpdate,
+       convertSQLQuery,
+       convertSQLDBUpdates,
        PersistenceStrategy(..),
        RelationalExpr,
        RelationalExprBase(..),
@@ -170,8 +170,8 @@ import qualified Network.RPC.Curryer.Server as RPC
 import Network.Socket (Socket, AddrInfo(..), getAddrInfo, defaultHints, AddrInfoFlag(..), SocketType(..), ServiceName, hostAddressToTuple, SockAddr(..))
 import GHC.Conc (unsafeIOToSTM)
 import ProjectM36.SQL.Select as SQL
-import ProjectM36.SQL.Update as SQL
-import ProjectM36.SQL.Convert
+import ProjectM36.SQL.DBUpdate as SQL
+import ProjectM36.SQL.Convert 
 
 type Hostname = String
 
@@ -1100,9 +1100,9 @@ getDDLHash sessionId (InProcessConnection conf) = do
         pure (ddlHash ctx graph)
 getDDLHash sessionId conn@RemoteConnection{} = remoteCall conn (GetDDLHash sessionId)
 
--- | Convert a SQL Select expression into a DataFrameExpr. Because the conversion process requires substantial database metadata access (such as retrieving types for various subexpressions), we cannot process SQL client-side. However, the underlying DBMS is completely unaware that the resultant DataFrameExpr has come from SQL.
-convertSQLSelect :: SessionId -> Connection -> Select -> IO (Either RelationalError DF.DataFrameExpr)
-convertSQLSelect sessionId (InProcessConnection conf) sel = do
+-- | Convert a SQL Query expression into a DataFrameExpr. Because the conversion process requires substantial database metadata access (such as retrieving types for various subexpressions), we cannot process SQL client-side. However, the underlying DBMS is completely unaware that the resultant DataFrameExpr has come from SQL.
+convertSQLQuery :: SessionId -> Connection -> Query -> IO (Either RelationalError DF.DataFrameExpr)
+convertSQLQuery sessionId (InProcessConnection conf) query = do
   let sessions = ipSessions conf
       graphTvar = ipTransactionGraph conf  
   atomically $ do
@@ -1115,13 +1115,13 @@ convertSQLSelect sessionId (InProcessConnection conf) sel = do
             reEnv = RE.mkRelationalExprEnv ctx transGraph
             typeF = optimizeAndEvalRelationalExpr reEnv
         -- convert SQL data into DataFrameExpr
-        case evalConvertM mempty (convertSelect typeF sel) of
+        case evalConvertM mempty (convertQuery typeF query) of
           Left err -> pure (Left (SQLConversionError err))
           Right dfExpr -> pure (Right dfExpr)
-convertSQLSelect sessionId conn@RemoteConnection{} sel = remoteCall conn (ConvertSQLSelect sessionId sel)
+convertSQLQuery sessionId conn@RemoteConnection{} q = remoteCall conn (ConvertSQLQuery sessionId q)
 
-convertSQLUpdate :: SessionId -> Connection -> SQL.Update -> IO (Either RelationalError DatabaseContextExpr)
-convertSQLUpdate sessionId (InProcessConnection conf) update = do
+convertSQLDBUpdates :: SessionId -> Connection -> [SQL.DBUpdate] -> IO (Either RelationalError DatabaseContextExpr)
+convertSQLDBUpdates sessionId (InProcessConnection conf) updates = do
   let sessions = ipSessions conf
       graphTvar = ipTransactionGraph conf  
   atomically $ do
@@ -1134,11 +1134,10 @@ convertSQLUpdate sessionId (InProcessConnection conf) update = do
             reEnv = RE.mkRelationalExprEnv ctx transGraph
             typeF = optimizeAndEvalRelationalExpr reEnv
         -- convert SQL data into DataFrameExpr
-        case evalConvertM mempty (convertUpdate typeF update) of
+        case evalConvertM mempty (convertDBUpdates typeF updates) of
           Left err -> pure (Left (SQLConversionError err))
           Right updateExpr -> pure (Right updateExpr)
-convertSQLUpdate sessionId conn@RemoteConnection{} up = remoteCall conn (ConvertSQLUpdate sessionId up)          
-
+convertSQLDBUpdates sessionId conn@RemoteConnection{} ups = remoteCall conn (ConvertSQLUpdates sessionId ups)   
 
 registeredQueriesAsRelation :: SessionId -> Connection -> IO (Either RelationalError Relation)
 registeredQueriesAsRelation sessionId (InProcessConnection conf) = do

src/lib/ProjectM36/Error.hs

@@ -171,6 +171,7 @@ data SQLError = NotSupportedError T.Text |
                 TableAliasMismatchError TableAlias |
                 UnexpectedTableNameError TableName |
                 UnexpectedColumnNameError ColumnName |
+                ColumnNamesMismatch (S.Set UnqualifiedColumnName) (S.Set UnqualifiedColumnName) | -- used for INSERT expressions
                 ColumnResolutionError ColumnName |
                 ColumnAliasResolutionError ColumnAlias |
                 UnexpectedRelationalExprError RelationalExpr |

src/lib/ProjectM36/SQL/Convert.hs

@@ -4,7 +4,10 @@ module ProjectM36.SQL.Convert where
 import ProjectM36.Base as B
 import ProjectM36.Error
 import ProjectM36.SQL.Select
-import ProjectM36.SQL.Update as SQL
+import ProjectM36.SQL.Insert as Insert
+import ProjectM36.SQL.DBUpdate
+import ProjectM36.SQL.Update as Update
+import ProjectM36.SQL.Delete as Delete
 import ProjectM36.RelationalExpression
 import ProjectM36.DataFrame (DataFrameExpr(..), AttributeOrderExpr(..), Order(..), usesDataFrameFeatures)
 import ProjectM36.AttributeNames as A
@@ -169,10 +172,10 @@ generateColumnAlias (TableAlias tAlias) attrName = do
             True
           _ -> False --some conflict, so loop
       firstAvailableName = find nameIsAvailable potentialNames
-  traceShowM ("generateColumnAlias scan"::String, tAlias, attrName, firstAvailableName)     
+--  traceShowM ("generateColumnAlias scan"::String, tAlias, attrName, firstAvailableName)     
   case firstAvailableName of
     Just (ColumnName [nam]) -> pure (ColumnAlias nam)
-    _ -> throwSQLE (ColumnResolutionError (ColumnName [attrName]))
+    _ -> throwSQLE $ ColumnResolutionError (ColumnName [attrName])
 
 -- | Insert another table into the TableContext. Returns an alias map of any columns which could conflict with column names already present in the TableContext so that they can be optionally renamed.
 insertTable :: TableAlias -> RelationalExpr -> Attributes -> ConvertM ColumnAliasMap
@@ -466,7 +469,7 @@ attributeNameForColumnName colName = do
           ColumnName [_, col] | col `A.isAttributeNameContained` rvattrs ->
                                 -- the column has not been aliased, so we presume it can be use the column name directly
                                 pure col
-          _ -> throwSQLE $ ColumnResolutionError colName
+          _ -> throwSQLE $ traceShow ("attrnameforcolname"::String, rvattrs, colName) $ ColumnResolutionError colName
 {-
 attributeNameForColumnName :: ColumnName -> ConvertM AttributeName
 attributeNameForColumnName colName = do
@@ -516,6 +519,27 @@ baseDFExpr = DataFrameExpr { convertExpr = MakeRelationFromExprs (Just []) (Tupl
                              orderExprs = [],
                              offset = Nothing,
                              limit = Nothing }
+
+falseDFExpr :: DataFrameExpr
+falseDFExpr = DataFrameExpr { convertExpr = MakeRelationFromExprs (Just []) (TupleExprs () []), --relationFalse 
+                             orderExprs = [],
+                             offset = Nothing,
+                             limit = Nothing }
+
+
+convertQuery :: TypeForRelExprF -> Query -> ConvertM DataFrameExpr
+convertQuery typeF (QuerySelect sel) = convertSelect typeF sel
+convertQuery typeF (QueryValues vals) = do
+  let convertTupleExprs tupVals = do
+        TupleExpr . M.fromList <$> mapM (\(c, sexpr) -> do
+                                            atomExpr <- convertScalarExpr typeF sexpr
+                                            pure ("attr_" <> T.pack (show c), atomExpr)
+                                        ) (zip [1::Int ..] tupVals)
+  tupleExprs <- mapM convertTupleExprs vals
+  pure (baseDFExpr { convertExpr = MakeRelationFromExprs Nothing (TupleExprs () tupleExprs) })
+convertQuery typeF (QueryTable tname) = do
+  rvName <- convertTableName tname
+  pure $ baseDFExpr { convertExpr = RelationVariable rvName () }
 
 convertSelect :: TypeForRelExprF -> Select -> ConvertM DataFrameExpr
 convertSelect typeF sel = do
@@ -1109,9 +1133,46 @@ convertUpdate typeF up = do
   restrictionExpr <- case mRestriction up of
                        Nothing -> pure TruePredicate
                        Just restriction -> convertWhereClause typeF restriction
-  rvname <- convertTableName (target up)
+  rvname <- convertTableName (Update.target up)
   pure (B.Update rvname atomMap restrictionExpr)
 
 convertTableName :: TableName -> ConvertM RelVarName
 convertTableName (TableName [tname]) = pure tname
 convertTableName t@TableName{} = throwSQLE (UnexpectedTableNameError t)
+
+convertDBUpdates :: TypeForRelExprF -> [DBUpdate] -> ConvertM DatabaseContextExpr
+convertDBUpdates typeF dbUpdates = MultipleExpr <$> mapM (convertDBUpdate typeF) dbUpdates
+
+convertDBUpdate :: TypeForRelExprF -> DBUpdate -> ConvertM DatabaseContextExpr
+convertDBUpdate typeF (UpdateUpdate up) = convertUpdate typeF up
+convertDBUpdate typeF (UpdateInsert ins) = convertInsert typeF ins
+convertDBUpdate typeF (UpdateDelete del) = convertDelete typeF del
+
+convertInsert :: TypeForRelExprF -> Insert -> ConvertM DatabaseContextExpr
+convertInsert typeF ins = do
+  dfExpr <- convertQuery typeF (source ins)
+  when (usesDataFrameFeatures dfExpr) $ throwSQLE (NotSupportedError "ORDER BY/LIMIT/OFFSET in subquery")
+  -- check that all columns are mentioned because Project:M36 does not support default columns
+  case typeF (convertExpr dfExpr) of
+    Left err -> throwSQLE (SQLRelationalError err)
+    Right rvExprType -> do
+      let rvExprAttrNames = A.attributeNamesList (attributes rvExprType)
+          insAttrNames = map convertUnqualifiedColumnName (targetColumns ins)
+          rvExprColNameSet = S.map UnqualifiedColumnName (S.fromList rvExprAttrNames)
+          insAttrColSet = S.fromList (targetColumns ins)
+      when (length rvExprAttrNames /= length insAttrNames) $ throwSQLE (ColumnNamesMismatch rvExprColNameSet insAttrColSet)
+      rvTarget <- convertTableName (Insert.target ins)
+      -- rename attributes rexpr via query/values to map to targetCol attrs
+      let insExpr = Rename (S.fromList (zip rvExprAttrNames insAttrNames)) (convertExpr dfExpr)
+      pure $ B.Insert rvTarget insExpr
+
+convertDelete :: TypeForRelExprF -> Delete.Delete -> ConvertM DatabaseContextExpr
+convertDelete typeF del = do
+  rvname <- convertTableName (Delete.target del)
+  let rv = RelationVariable rvname ()
+  case typeF rv of
+    Left err -> throwSQLE (SQLRelationalError err)
+    Right typeRel -> do
+      insertTable (TableAlias rvname) rv (attributes typeRel)
+      res <- convertWhereClause typeF (restriction del)
+      pure (B.Delete rvname res)

src/lib/ProjectM36/SQL/DBUpdate.hs

@@ -0,0 +1,17 @@
+{-# LANGUAGE DeriveGeneric, DerivingStrategies, DerivingVia, DeriveAnyClass #-}
+module ProjectM36.SQL.DBUpdate where
+import ProjectM36.SQL.Update
+import ProjectM36.SQL.Insert
+import ProjectM36.SQL.Delete
+import Control.DeepSeq
+import Codec.Winery
+import GHC.Generics
+
+-- | represents any SQL expression which can change the current transaction state such as
+data DBUpdate = UpdateUpdate Update |
+                UpdateInsert Insert |
+                UpdateDelete Delete
+  deriving (Show, Eq, Generic, NFData)
+  deriving Serialise via WineryVariant DBUpdate
+
+

src/lib/ProjectM36/SQL/Delete.hs

@@ -0,0 +1,12 @@
+{-# LANGUAGE DeriveGeneric, DerivingStrategies, DerivingVia, DeriveAnyClass #-}
+module ProjectM36.SQL.Delete where
+import ProjectM36.SQL.Select
+import Control.DeepSeq
+import Codec.Winery
+import GHC.Generics
+
+data Delete = Delete { target :: TableName,
+                       restriction :: RestrictionExpr
+                     }
+  deriving (Show, Eq, Generic, NFData)
+  deriving Serialise via WineryRecord Delete