diff --git a/app/Airsequel.hs b/app/Airsequel.hs index 3b59a8a..21e1c01 100644 --- a/app/Airsequel.hs +++ b/app/Airsequel.hs @@ -65,7 +65,7 @@ import Text.RawString.QQ (r) import Data.Aeson.Types (parseEither) import Types (ExtendedRepo (..), SaveStrategy (..)) -import Utils (loadAsWriteToken, loadDbEndpoint, var) +import Utils (encodeToText, loadAirsWriteToken, loadDbEndpoint, var) setRequestFields :: Text -> Text -> Request -> Request @@ -104,7 +104,7 @@ upsertRepoQuery utc extendedRepo rowidMb = github_id: <> owner: "<>" name: "<>" - description: "<>" + description: <> homepage: "<>" language: "<>" stargazers_count: <> @@ -131,6 +131,11 @@ upsertRepoQuery utc extendedRepo rowidMb = } ) { affected_rows + returning { + owner + name + rowid + } } } |] @@ -143,7 +148,7 @@ upsertRepoQuery utc extendedRepo rowidMb = & var "github_id" (repo.repoId & GH.untagId & show) & var "owner" (repo.repoOwner.simpleOwnerLogin & untagName) & var "name" (repo.repoName & untagName) - & var "description" (repo.repoDescription & fromMaybe "") + & var "description" (repo.repoDescription & fromMaybe "" & encodeToText) & var "homepage" (repo.repoHomepage & fromMaybe "") & var "language" @@ -163,9 +168,11 @@ upsertRepoQuery utc extendedRepo rowidMb = -- | Get rowid of a repo with the specified GitHub ID -getRowid :: Manager -> Text -> Text -> Int -> IO (Maybe Int) -getRowid manager dbEndpoint airseqWriteToken github_id = do +getRowid :: Manager -> Text -> Text -> ExtendedRepo -> IO (Maybe Int) +getRowid manager dbEndpoint airseqWriteToken extendedRepo = do let + githubId = extendedRepo.core.repoId & GH.untagId + getRowidQuery :: Text getRowidQuery = [r| @@ -179,7 +186,7 @@ getRowid manager dbEndpoint airseqWriteToken github_id = do } } |] - & var "github_id" (show github_id) + & var "github_id" (show githubId) initialGetRowidRequest <- parseRequest $ T.unpack dbEndpoint @@ -194,25 +201,32 @@ getRowid manager dbEndpoint airseqWriteToken github_id = do (getRowidResponse.responseStatus.statusCode /= 200) (putErrText $ show getRowidResponse.responseBody) - let rowidResult :: Either [P.Char] Int = - ( getRowidResponse.responseBody - & eitherDecode - :: Either [P.Char] Object - ) - >>= ( \gqlRes -> - P.flip parseEither gqlRes $ \gqlResObj -> do - gqlData <- gqlResObj .: "data" - gqlData .: "repos" - ) - >>= ( \case - [] -> Left "Repo is not in Airsequel yet" - [repo :: Object] -> parseEither (.: "rowid") repo - _ -> - Left $ - "Error: Repo with GitHub ID \"" - <> show github_id - <> "\" is not unique in Airsequel" - ) + let + msgBase = + "Repo \"" + <> (extendedRepo.core.repoOwner.simpleOwnerLogin & untagName) + <> "/" + <> (extendedRepo.core.repoName & untagName) + <> "\" is not" + + rowidResult :: Either [P.Char] Int = + ( getRowidResponse.responseBody + & eitherDecode + :: Either [P.Char] Object + ) + >>= ( \gqlRes -> + P.flip parseEither gqlRes $ \gqlResObj -> do + gqlData <- gqlResObj .: "data" + gqlData .: "repos" + ) + >>= ( \case + [] -> Left $ T.unpack $ msgBase <> " in Airsequel yet" + [repo :: Object] -> parseEither (.: "rowid") repo + _ -> + Left $ + T.unpack $ + "Error: " <> msgBase <> " unique in Airsequel" + ) case rowidResult of Left err -> do @@ -232,7 +246,7 @@ via a POST request executed by http-client saveRepoInAirsequel :: SaveStrategy -> ExtendedRepo -> IO () saveRepoInAirsequel saveStrategy extendedRepo = do dbEndpoint <- loadDbEndpoint - airseqWriteToken <- loadAsWriteToken + airseqWriteToken <- loadAirsWriteToken manager <- newManager tlsManagerSettings @@ -247,7 +261,7 @@ saveRepoInAirsequel saveStrategy extendedRepo = do manager dbEndpoint airseqWriteToken - (extendedRepo.core.repoId & GH.untagId) + extendedRepo else pure Nothing initialInsertRequest <- parseRequest $ T.unpack dbEndpoint @@ -277,6 +291,11 @@ deleteRepo manager dbEndpoint airseqWriteToken extendedRepo = do } ) { affected_rows + returning { + owner + name + rowid + } } } |] diff --git a/app/Main.hs b/app/Main.hs index dc8808c..3e2442d 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -1,8 +1,8 @@ {-# LANGUAGE QuasiQuotes #-} {-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Monoid law, left identity" #-} {-# HLINT ignore "Use maybe" #-} -{-# HLINT ignore "Use unless" #-} module Main where @@ -18,8 +18,10 @@ import Protolude ( encodeUtf8, find, fromMaybe, + headMay, lastMay, mapM_, + mempty, pure, putErrText, putText, @@ -29,6 +31,7 @@ import Protolude ( ($), (&), (.), + (<$>), (<&>), (<>), (>>=), @@ -46,7 +49,7 @@ import Data.List (lookup) import Data.Text qualified as T import GHC.Base (String) import GitHub qualified as GH -import GitHub.Endpoints.Activity.Starring as GH (Repo, untagName) +import GitHub.Endpoints.Activity.Starring as GH (Auth (OAuth), Repo, untagName) import GitHub.Internal.Prelude (fromString) import Network.HTTP.Client ( RequestBody (RequestBodyLBS), @@ -64,6 +67,21 @@ import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Link (href, parseLinkHeaderBS) import Network.HTTP.Link.Types (Link, LinkParam (..), linkParams) import Network.URI (URI) +import Options.Applicative ( + Parser, + argument, + command, + execParser, + fullDesc, + header, + helper, + hsubparser, + info, + metavar, + progDesc, + str, + (<**>), + ) import Text.RawString.QQ (r) import Airsequel (saveRepoInAirsequel) @@ -71,6 +89,34 @@ import Types (ExtendedRepo (..), GqlResponse (..), SaveStrategy (..)) import Utils (loadGitHubToken, mapMSequentially, var) +-- TODO: Add CLI flag to choose between OverwriteRepo and AddRepo +data CliCmd + = -- | Upload a single repo + Upload Text + | -- | Search for repos and upload them + Search + + +commands :: Parser CliCmd +commands = do + let + upload :: Parser CliCmd + upload = Upload <$> argument str (metavar "REPO_SLUG") + + search :: Parser CliCmd + search = pure Search + + hsubparser + ( mempty + <> command + "upload" + (info upload (progDesc "Upload a single repo")) + <> command + "search" + (info search (progDesc "Search for and upload several repos")) + ) + + formatRepo :: ExtendedRepo -> Text formatRepo extendedRepo = let @@ -164,8 +210,13 @@ getNumberOfCommits ghTokenMb repo = do -} loadAndSaveRepo :: Maybe Text -> SaveStrategy -> Text -> Text -> IO () loadAndSaveRepo ghTokenMb saveStrategy owner name = do + let + uploadFunc = case ghTokenMb of + Nothing -> GH.github' + Just ghToken -> GH.github (OAuth (encodeUtf8 ghToken)) + repoResult <- - GH.github' + uploadFunc GH.repositoryR (fromString $ T.unpack owner) (fromString $ T.unpack name) @@ -195,17 +246,17 @@ getGhHeaders tokenMb = Nothing -> [] -execGqlQuery - :: Text - -> Maybe Text +execGithubGqlQuery + :: Maybe Text -> Text -> Maybe Text -> [ExtendedRepo] -> IO [ExtendedRepo] -execGqlQuery apiEndpoint ghTokenMb query nextCursorMb initialRepos = do +execGithubGqlQuery ghTokenMb query nextCursorMb initialRepos = do manager <- newManager tlsManagerSettings - initialRequest <- parseRequest $ T.unpack apiEndpoint + initialRequest <- parseRequest $ T.unpack "https://api.github.com/graphql" + let request = initialRequest { method = "POST" @@ -227,7 +278,9 @@ execGqlQuery apiEndpoint ghTokenMb query nextCursorMb initialRepos = do ) ] } + response <- httpLbs request manager + let gqlResult :: Either String GqlResponse = response.responseBody & eitherDecode @@ -271,8 +324,7 @@ execGqlQuery apiEndpoint ghTokenMb query nextCursorMb initialRepos = do case gqlResponse.nextCursorMb of Nothing -> pure $ initialRepos <> extendedRepos Just nextCursor -> do - execGqlQuery - apiEndpoint + execGithubGqlQuery ghTokenMb query (Just nextCursor) @@ -317,50 +369,77 @@ loadAndSaveReposViaSearch githubToken searchQuery numRepos = do & var "searchQuery" searchQuery & var "numRepos" (show numRepos) - execGqlQuery - "https://api.github.com/graphql" + execGithubGqlQuery githubToken gqlQUery Nothing [] -main :: IO () -main = do +-- | Function to handle the execution of commands +run :: CliCmd -> IO () +run cliCmd = do ghTokenMb <- loadGitHubToken - -- TODO: Add CLI flag to load and save a single repo - -- loadAndSaveRepo ghTokenMb OverwriteRepo "Airsequel" "SQLiteDAV" + case cliCmd of + Upload repoSlug -> do + let + fragments = repoSlug & T.splitOn "/" + ownerMb = fragments & headMay + nameMb = fragments & lastMay + + case ownerMb of + Nothing -> putErrText "Error: Repo owner is missing" + Just owner -> + case nameMb of + Nothing -> putErrText "Error: Repo name is missing" + Just name -> + loadAndSaveRepo + ghTokenMb + OverwriteRepo + owner + name + Search -> do + -- Good filter options: + -- language:haskell + -- stars:>=10 + -- stars:10..50 + -- sort:updated-desc + -- sort:stars-asc + -- archived:true + let searchQuery = + [r| + language:haskell + stars:56..76 + sort:stars-desc + |] + & T.replace "\n" " " + & T.strip + + repos <- loadAndSaveReposViaSearch ghTokenMb searchQuery 20 + + putText $ "Found " <> show (P.length repos) <> " repos:" + repos + <&> ( \repo -> + (repo.core.repoOwner.simpleOwnerLogin & untagName) + <> ("/" :: Text) + <> (repo.core.repoName & untagName) + <> (" " :: Text) + <> show repo.commitsCount + ) + & mapM_ putText + + pure () - -- TODO: Add CLI flag to choose between OverwriteRepo and AddRepo - -- Good filter options: - -- language:haskell - -- stars:>=10 - -- stars:10..50 - -- sort:updated-desc - -- sort:stars-asc - -- archived:true - let searchQuery = - [r| - language:haskell - stars:14..15 - sort:stars-desc - |] - & T.replace "\n" " " - & T.strip - - repos <- loadAndSaveReposViaSearch ghTokenMb searchQuery 2 - - putText $ "Found " <> show (P.length repos) <> " repos:" - repos - <&> ( \repo -> - (repo.core.repoOwner.simpleOwnerLogin & untagName) - <> ("/" :: Text) - <> (repo.core.repoName & untagName) - <> (" " :: Text) - <> show repo.commitsCount - ) - & mapM_ putText - - pure () +main :: IO () +main = do + let opts = + info + (commands <**> helper) + ( fullDesc + <> progDesc "Upload repo metadata to Airsequel" + <> header "repos-uploader" + ) + + execParser opts >>= run diff --git a/app/Utils.hs b/app/Utils.hs index 0cf1465..8125464 100644 --- a/app/Utils.hs +++ b/app/Utils.hs @@ -1,5 +1,6 @@ module Utils ( - loadAsWriteToken, + encodeToText, + loadAirsWriteToken, loadDbEndpoint, loadDbId, loadGitHubToken, @@ -11,29 +12,50 @@ where import Protolude ( IO, Int, - Maybe, + Maybe (..), Text, - fromMaybe, + decodeUtf8, liftIO, mapM, pure, + putErrText, ($), (*), - (<&>), + (.), (<*), (<>), ) -import Control.Arrow ((>>>)) import Control.Concurrent (threadDelay) +import Data.Aeson (ToJSON, encode) +import Data.ByteString.Lazy (toStrict) import Data.Text qualified as T import System.Environment (lookupEnv) +import System.Exit (die) + + +encodeToText :: (ToJSON a) => a -> Text +encodeToText = + decodeUtf8 . toStrict . encode + + +lookupEnvOrDie :: Text -> IO Text +lookupEnvOrDie envVarName = do + envVarMb <- lookupEnv (T.unpack envVarName) + case envVarMb of + Just envVar -> pure $ T.pack envVar + Nothing -> do + die $ + T.unpack $ + "ERROR: " + <> envVarName + <> " environment variable must be set" -- | The ID of the Airsequel database loaded from the environment loadDbId :: IO Text loadDbId = - lookupEnv "AIRSEQUEL_DB_ID" <&> (fromMaybe "" >>> T.pack) + lookupEnvOrDie "AIRSEQUEL_DB_ID" loadDbEndpoint :: IO Text @@ -42,14 +64,22 @@ loadDbEndpoint = do pure $ "https://www.airsequel.com/dbs/" <> dbId <> "/graphql" -loadAsWriteToken :: IO Text -loadAsWriteToken = - lookupEnv "AIRSEQUEL_API_TOKEN" <&> (fromMaybe "" >>> T.pack) +loadAirsWriteToken :: IO Text +loadAirsWriteToken = + lookupEnvOrDie "AIRSEQUEL_API_TOKEN" loadGitHubToken :: IO (Maybe Text) -loadGitHubToken = - lookupEnv "GITHUB_TOKEN" <&> (<&> T.pack) +loadGitHubToken = do + ghTokenMb <- lookupEnv "GITHUB_TOKEN" + case ghTokenMb of + Nothing -> do + putErrText + "WARNING: Without a GITHUB_TOKEN environment variable \ + \all requests to GitHub will be unauthenticated." + pure Nothing + Just token -> + pure $ Just $ T.pack token -- | Replaces a variable in a string with a value diff --git a/package.yaml b/package.yaml index ac90b2f..72a6271 100644 --- a/package.yaml +++ b/package.yaml @@ -17,16 +17,19 @@ extra-source-files: dependencies: - aeson - base - - protolude - - raw-strings-qq - - text + - bytestring - http-link-header - http-types - network-uri + - optparse-applicative + - protolude + - raw-strings-qq + - text - time default-extensions: - ImportQualifiedPost + - LambdaCase - NoImplicitPrelude - OverloadedRecordDot - OverloadedStrings diff --git a/repos-uploader.cabal b/repos-uploader.cabal index d25b17b..7b747b8 100644 --- a/repos-uploader.cabal +++ b/repos-uploader.cabal @@ -4,7 +4,7 @@ cabal-version: 2.2 -- -- see: https://github.com/sol/hpack -- --- hash: 814ea5e24185d06d45ae7fbede4400077c963f0056a641ff8e843da2f8062efb +-- hash: 63fc3e8744bf308b5a7b0c2268b75bb440d310bd2dc36e72791d893e12b745c3 name: repos-uploader version: 0.0.0.0 @@ -32,6 +32,7 @@ library source default-extensions: ImportQualifiedPost + LambdaCase NoImplicitPrelude OverloadedRecordDot OverloadedStrings @@ -40,12 +41,14 @@ library build-depends: aeson , base + , bytestring , github , http-client , http-client-tls , http-link-header , http-types , network-uri + , optparse-applicative , protolude , raw-strings-qq , text @@ -65,6 +68,7 @@ executable repos-uploader app default-extensions: ImportQualifiedPost + LambdaCase NoImplicitPrelude OverloadedRecordDot OverloadedStrings @@ -73,12 +77,14 @@ executable repos-uploader build-depends: aeson , base + , bytestring , github , http-client , http-client-tls , http-link-header , http-types , network-uri + , optparse-applicative , protolude , raw-strings-qq , text