diff --git a/airput.cabal b/airput.cabal index a51723d..1d0876d 100644 --- a/airput.cabal +++ b/airput.cabal @@ -39,6 +39,8 @@ library aeson , base , bytestring + , directory + , filepath , http-client , http-client-tls , http-link-header @@ -49,12 +51,14 @@ library , raw-strings-qq , text , time + , vector default-language: GHC2021 executable airput main-is: Main.hs other-modules: Airsequel + FileUploader Types Utils Paths_airput @@ -74,6 +78,8 @@ executable airput aeson , base , bytestring + , directory + , filepath , http-client , http-client-tls , http-link-header @@ -84,4 +90,5 @@ executable airput , raw-strings-qq , text , time + , vector default-language: GHC2021 diff --git a/app/FileUploader.hs b/app/FileUploader.hs new file mode 100644 index 0000000..90ed00d --- /dev/null +++ b/app/FileUploader.hs @@ -0,0 +1,213 @@ +module FileUploader (uploadFiles) +where + +import Protolude ( + Either (Left, Right), + IO, + Maybe (Just), + Show, + filterM, + flip, + pure, + putErrLn, + putStrLn, + ($), + (&), + (.), + (/=), + (<&>), + (<>), + (||), + ) + +import Control.Arrow ((>>>)) +import Control.Monad ((=<<)) +import Data.Aeson (encode, object, (.=)) +import Data.Aeson qualified as Aeson +import Data.Aeson.KeyMap (lookup) +import Data.ByteString qualified as BS +import Data.ByteString.Char8 (unpack) +import Data.ByteString.Lazy qualified as BSL +import Data.Int (Int) +import Data.Text (Text) +import Data.Text qualified as T +import Data.Vector qualified as Vector (toList) +import GHC.Generics (Generic) +import Network.HTTP.Client ( + Request (method, requestBody, requestHeaders), + RequestBody (RequestBodyLBS), + Response (responseBody, responseStatus), + httpLbs, + newManager, + parseRequest, + ) +import Network.HTTP.Client.MultipartFormData (formDataBody, partFileSource) +import Network.HTTP.Client.TLS (tlsManagerSettings) +import Network.HTTP.Types (Status (statusMessage), methodPut) +import Protolude qualified as P +import System.Directory (doesFileExist, listDirectory, makeAbsolute) +import System.FilePath (FilePath, takeBaseName, takeExtension, ()) + + +-- | Escape double quotes in SQL strings +escDoubleQuotes :: Text -> Text +escDoubleQuotes = + T.replace "\"" "\"\"" + + +-- | Quote a keyword in an SQL query +quoteKeyword :: Text -> Text +quoteKeyword keyword = + keyword + & escDoubleQuotes + & (\word -> "\"" <> word <> "\"") + + +-- | Escape single quotes in SQL strings +escSingleQuotes :: Text -> Text +escSingleQuotes = + T.replace "'" "''" + + +-- | Quote literal text in an SQL query +quoteText :: Text -> Text +quoteText keyword = + keyword + & escSingleQuotes + & (\word -> "'" <> word <> "'") + + +data FileData = FileData + { name :: FilePath + , song_id :: Int + , filetype :: Text + } + deriving (Generic, Show) + + +getFilesSorted :: FilePath -> IO [FilePath] +getFilesSorted path = do + filePaths <- listDirectory path + filePaths + & filterM (doesFileExist . (path )) + <&> ( P.filter (/= ".DS_Store") + >>> P.sort + ) + + +createSQLQuery :: Text -> FileData -> Text +createSQLQuery tableName fileData = + ("INSERT INTO " <> quoteKeyword tableName <> " (name, filetype) ") + <> ( "VALUES (" + <> quoteText (T.pack $ takeBaseName fileData.name) + <> ", " + <> quoteText fileData.filetype + <> ") " + ) + <> "RETURNING rowid" + + +uploadFiles :: Text -> Text -> Text -> [FilePath] -> IO () +uploadFiles domain dbId tableName paths = do + manager <- newManager tlsManagerSettings + + fileLists <- P.forM paths $ \filePath -> do + isFile <- doesFileExist filePath + + if isFile + then pure [filePath] + else getFilesSorted filePath + + P.forM_ (P.concat fileLists) $ \fileName -> do + let + fileData = + FileData + { name = fileName + , song_id = 0 + , filetype = T.drop 1 $ T.pack $ takeExtension fileName + } + url :: Text = domain <> "/dbs/" <> dbId <> "/sql" + + initialRequest <- parseRequest $ T.unpack url + + let + query = createSQLQuery tableName fileData + sqlRequest = + initialRequest + { method = "POST" + , requestBody = RequestBodyLBS $ encode $ object ["query" .= query] + , requestHeaders = [("Content-Type", "application/json")] + } + + sqlResponse <- httpLbs sqlRequest manager + + let resBody = sqlResponse.responseBody & BSL.toStrict + + if (sqlResponse.responseStatus.statusMessage /= "OK") + || ( ("error" `BS.isInfixOf` resBody) + P.&& P.not ("errors\":[]" `BS.isInfixOf` resBody) + ) + then + putErrLn $ + "ERROR:\n" + <> ("File entry \"" <> fileName <> "\" could not be inserted.\n") + <> P.show sqlResponse + else + putStrLn $ + "Inserted file \"" <> fileName <> "\" into SQLite database." + + let + body = sqlResponse.responseBody & BSL.toStrict + + -- Extract rowid from json response with aeson + rowidResult :: Either Text BS.ByteString + rowidResult = + case Aeson.decode $ BSL.fromStrict body of + Just (Aeson.Object obj) -> case obj & lookup "rows" of + Just (Aeson.Array x) -> case Vector.toList x of + (Aeson.Object y) : _ -> case y & lookup "rowid" of + Just (Aeson.Number z) -> Right $ BSL.toStrict $ encode z + _ -> Left "Could not find rowid in row" + _ -> Left "Rows is not an array" + _ -> Left "Could not find rows in response" + _ -> Left "Could not decode response" + + case rowidResult of + Left err -> do + putErrLn err + pure () + Right rowid -> do + -- Upload file + initialFileRequest <- + parseRequest $ + T.unpack $ + domain + <> "/api/dbs/" + <> dbId + <> "/tables/" + <> tableName + <> "/columns/content/files/rowid/" + <> (rowid & unpack & T.pack) + + filePathAbs <- makeAbsolute fileName + + let + fileRequest = + formDataBody + [partFileSource "file" filePathAbs] + initialFileRequest + + fileRes <- + flip httpLbs manager + =<< (fileRequest <&> (\req -> req{method = methodPut})) + + if (fileRes.responseStatus.statusMessage /= "OK") + || ("error" `BS.isInfixOf` (fileRes.responseBody & BSL.toStrict)) + then + putErrLn $ + "ERROR:\n" + <> ("File \"" <> fileName <> "\" could not be uploaded.\n") + <> P.show fileRes + else + putStrLn $ + "Uploaded file \"" <> fileName <> "\"" diff --git a/app/Main.hs b/app/Main.hs index fd4792f..dd5a4f0 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -7,16 +7,13 @@ module Main where import Protolude ( - Bool (..), Either (Left, Right), IO, Int, Integer, Maybe (..), Text, - elem, encodeUtf8, - find, fromMaybe, headMay, lastMay, @@ -30,24 +27,23 @@ import Protolude ( when, ($), (&), - (.), (<$>), (<&>), + (<*>), (<>), (>), (>>=), ) import Protolude qualified as P +import Control.Arrow ((>>>)) import Data.Aeson (Value (String), eitherDecode, encode, object, (.=)) import Data.Aeson.KeyMap (KeyMap) import Data.Aeson.KeyMap qualified as KeyMap -import Data.List (lookup) import Data.Text qualified as T import GHC.Base (String) import Network.HTTP.Client ( RequestBody (RequestBodyLBS), - Response (responseHeaders), httpLbs, method, newManager, @@ -57,9 +53,6 @@ import Network.HTTP.Client ( responseBody, ) import Network.HTTP.Client.TLS (tlsManagerSettings) -import Network.HTTP.Link (href, parseLinkHeaderBS) -import Network.HTTP.Link.Types (Link, LinkParam (..), linkParams) -import Network.URI (URI) import Options.Applicative ( Parser, argument, @@ -67,26 +60,39 @@ import Options.Applicative ( execParser, fullDesc, headerDoc, + help, helper, hsubparser, info, + long, metavar, progDesc, progDescDoc, + showDefault, + some, str, + strOption, + value, (<**>), ) +import Options.Applicative.Help.Pretty (vsep) import Text.RawString.QQ (r) import Airsequel (saveReposInAirsequel) -import Control.Arrow ((>>>)) -import Options.Applicative.Help.Pretty (vsep) +import FileUploader (uploadFiles) import Types (GqlRepoRes (..), Repo (..), SaveStrategy (..)) import Utils (loadGitHubToken) data CliCmd - = -- | Upload metadata for a single GitHub repo + = -- | Upload files + FileUpload + { domain :: [P.Char] + , dbId :: [P.Char] + , tableName :: [P.Char] + , paths :: [P.FilePath] + } + | -- | Upload metadata for a single GitHub repo GithubUpload Text | -- | Search for GitHub repos and upload their metadata GithubSearch [Text] @@ -95,14 +101,47 @@ data CliCmd commands :: Parser CliCmd commands = do let + fileUpload :: Parser CliCmd + fileUpload = + FileUpload + <$> strOption + ( long "domain" + <> metavar "DOMAIN_NAME" + <> help "Domain to upload files to" + <> showDefault + <> value "https://www.airsequel.com" + ) + <*> strOption + ( long "dbid" + <> metavar "DATABASE_ID" + <> help "Database ID to upload files to" + ) + <*> strOption + ( long "tablename" + <> metavar "TABLE_NAME" + <> help "Table name to upload files to" + ) + <*> some (argument str (metavar "FILE/DIR...")) + githubUpload :: Parser CliCmd - githubUpload = GithubUpload <$> argument str (metavar "REPO_SLUG") + githubUpload = + GithubUpload <$> argument str (metavar "REPO_SLUG") githubSearch :: Parser CliCmd - githubSearch = GithubSearch <$> many (argument str (metavar "SEARCH_QUERY")) + githubSearch = + GithubSearch <$> many (argument str (metavar "SEARCH_QUERY")) hsubparser ( mempty + <> command + "upload" + ( info + fileUpload + ( progDesc + "Upload files to a database via the REST API. \ + \Expects 3 columns: `name`, `filetype`, and `content`." + ) + ) <> command "github-upload" ( info @@ -136,22 +175,6 @@ commands = do ) --- | Query @Link@ header with @rel=last@ from the request headers -getLastUrl :: Response a -> Maybe URI -getLastUrl req = do - let - isRelNext :: Link uri -> Bool - isRelNext = elem relNextLinkParam . linkParams - - relNextLinkParam :: (LinkParam, Text) - relNextLinkParam = (Rel, "last") - - linkHeader <- lookup "Link" (responseHeaders req) - links <- parseLinkHeaderBS linkHeader - nextURI <- find isRelNext links - pure $ href nextURI - - {-| Loads a single repo from GitHub, adds number of commits, | and saves it to Airsequel -} @@ -371,10 +394,12 @@ loadAndSaveReposViaSearch ghTokenMb searchQuery numRepos afterMb = do -- | Function to handle the execution of commands run :: CliCmd -> IO () run cliCmd = do - ghTokenMb <- loadGitHubToken - case cliCmd of + FileUpload{domain, dbId, tableName, paths} -> do + uploadFiles (T.pack domain) (T.pack dbId) (T.pack tableName) paths + -- GithubUpload repoSlug -> do + ghTokenMb <- loadGitHubToken let fragments = repoSlug & T.splitOn "/" ownerMb = fragments & headMay @@ -393,7 +418,9 @@ run cliCmd = do owner name pure () + -- GithubSearch searchQueries -> do + ghTokenMb <- loadGitHubToken let searchQueriesNorm = searchQueries <&> (T.replace "\n" " " >>> T.strip) allRepos <- P.forM searchQueriesNorm $ \searchQueryNorm -> do diff --git a/makefile b/makefile new file mode 100644 index 0000000..75d03f4 --- /dev/null +++ b/makefile @@ -0,0 +1,20 @@ +.PHONY: help +help: makefile + @tail -n +4 makefile | grep ".PHONY" + + +.PHONY: test +test: + stack test airput --fast + + +.PHONY: install +install: + stack build airput \ + --fast \ + --copy-bins + + +.PHONY: clean +clean: + stack purge diff --git a/package.yaml b/package.yaml index 4aafdbe..d1408d1 100644 --- a/package.yaml +++ b/package.yaml @@ -16,6 +16,8 @@ dependencies: - aeson - base - bytestring + - directory + - filepath - http-client - http-client-tls - http-link-header @@ -26,6 +28,7 @@ dependencies: - raw-strings-qq - text - time + - vector default-extensions: - ImportQualifiedPost