From a31d51ea83bf37bdd4c2daf7d49b65ed36a2150c Mon Sep 17 00:00:00 2001 From: Adrian Sieber Date: Fri, 5 Jan 2024 20:27:52 +0000 Subject: [PATCH] Use proper GraphQL variables for all GraphQL queries --- app/Airsequel.hs | 235 ++++++++++++++++++++++++----------------------- app/Main.hs | 77 +++++++++------- app/Utils.hs | 17 ---- 3 files changed, 161 insertions(+), 168 deletions(-) diff --git a/app/Airsequel.hs b/app/Airsequel.hs index 21e1c01..40d5f57 100644 --- a/app/Airsequel.hs +++ b/app/Airsequel.hs @@ -41,6 +41,8 @@ import Data.Aeson ( (.:), (.=), ) +import Data.Aeson.KeyMap qualified as KeyMap +import Data.Aeson.Types (parseEither) import Data.Text qualified as T import Data.Time (getCurrentTime) import Data.Time.Format.ISO8601 (iso8601Show) @@ -63,13 +65,12 @@ import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Types (statusCode) import Text.RawString.QQ (r) -import Data.Aeson.Types (parseEither) import Types (ExtendedRepo (..), SaveStrategy (..)) -import Utils (encodeToText, loadAirsWriteToken, loadDbEndpoint, var) +import Utils (loadAirsWriteToken, loadDbEndpoint) -setRequestFields :: Text -> Text -> Request -> Request -setRequestFields airseqWriteToken query req = +setRequestFields :: Text -> Text -> Object -> Request -> Request +setRequestFields airseqWriteToken query variables req = req { method = "POST" , requestHeaders = @@ -79,92 +80,74 @@ setRequestFields airseqWriteToken query req = , requestBody = RequestBodyLBS $ encode $ - object ["query" .= query] + object + [ "query" .= query + , "variables" .= variables + ] } --- | Insert repos in Airsequel and update them if rowid is already assigned -upsertRepoQuery :: Text -> ExtendedRepo -> Maybe Int -> Text -upsertRepoQuery utc extendedRepo rowidMb = - let - repo = extendedRepo.core - commitsCount = extendedRepo.commitsCount - getTimestamp field = - repo - & field - <&> iso8601Show - & fromMaybe "" - & T.pack - in - [r| - mutation { - insert_repos( - objects: [{ - <> - github_id: <> - owner: "<>" - name: "<>" - description: <> - homepage: "<>" - language: "<>" - stargazers_count: <> - open_issues_count: <> - <> - is_archived: <> - created_utc: "<>" - updated_utc: "<>" - crawled_utc: "<>" - }] - on_conflict: { - constraint: rowid - update_columns: [ - description - homepage - language - stargazers_count - open_issues_count - commits_count - is_archived - updated_utc - crawled_utc - ] - } - ) { - affected_rows - returning { - owner - name - rowid - } +-- | Insert repos in Airsequel or update them if rowid is already assigned +upsertRepoQuery :: Text +upsertRepoQuery = do + [r| + mutation InsertRepo ( + $rowid: Int + $github_id: Int! + $owner: String! + $name: String! + $description: String + $homepage: String + $language: String + $stargazers_count: Int + $open_issues_count: Int + $commits_count: Int + $is_archived: Boolean + $created_utc: String + $updated_utc: String + $crawled_utc: String! + ) { + insert_repos( + objects: [{ + rowid: $rowid + github_id: $github_id + owner: $owner + name: $name + description: $description + homepage: $homepage + language: $language + stargazers_count: $stargazers_count + open_issues_count: $open_issues_count + commits_count: $commits_count + is_archived: $is_archived + created_utc: $created_utc + updated_utc: $updated_utc + crawled_utc: $crawled_utc + }] + on_conflict: { + constraint: rowid + update_columns: [ + description + homepage + language + stargazers_count + open_issues_count + commits_count + is_archived + updated_utc + crawled_utc + ] } + ) { + affected_rows + returning { + owner + name + rowid + } } - |] - & var - "rowid" - ( case rowidMb of - Just rowid -> "rowid: " <> show rowid - Nothing -> "" - ) - & var "github_id" (repo.repoId & GH.untagId & show) - & var "owner" (repo.repoOwner.simpleOwnerLogin & untagName) - & var "name" (repo.repoName & untagName) - & var "description" (repo.repoDescription & fromMaybe "" & encodeToText) - & var "homepage" (repo.repoHomepage & fromMaybe "") - & var - "language" - (repo.repoLanguage <&> GH.getLanguage & fromMaybe "") - & var "stargazers_count" (repo.repoStargazersCount & show) - & var "open_issues_count" (repo.repoOpenIssuesCount & show) - & var - "commits_count" - ( case commitsCount of - Just count -> "commits_count: " <> show count - Nothing -> "" - ) - & var "is_archived" (repo.repoArchived & show & T.toLower) - & var "created_utc" (getTimestamp GH.repoCreatedAt) - & var "updated_utc" (getTimestamp GH.repoUpdatedAt) - & var "crawled_utc" utc + } + |] -- | Get rowid of a repo with the specified GitHub ID @@ -176,17 +159,14 @@ getRowid manager dbEndpoint airseqWriteToken extendedRepo = do getRowidQuery :: Text getRowidQuery = [r| - query GetRowid { + query GetRowid($github_id: Int!) { repos( - filter: { - github_id: { eq: <> } - } + filter: { github_id: { eq: $github_id } } ) { rowid } } |] - & var "github_id" (show githubId) initialGetRowidRequest <- parseRequest $ T.unpack dbEndpoint @@ -194,6 +174,7 @@ getRowid manager dbEndpoint airseqWriteToken extendedRepo = do setRequestFields airseqWriteToken getRowidQuery + (KeyMap.fromList ["github_id" .= githubId]) initialGetRowidRequest getRowidResponse <- httpLbs getRowidRequest manager @@ -265,11 +246,32 @@ saveRepoInAirsequel saveStrategy extendedRepo = do else pure Nothing initialInsertRequest <- parseRequest $ T.unpack dbEndpoint - let insertRequest = - setRequestFields - airseqWriteToken - (upsertRepoQuery now extendedRepo rowidMb) - initialInsertRequest + + let + variables = + [ "rowid" .= rowidMb + , "github_id" .= (extendedRepo.core.repoId & GH.untagId) + , "owner" .= (extendedRepo.core.repoOwner.simpleOwnerLogin & untagName) + , "name" .= (extendedRepo.core.repoName & untagName) + , "description" .= extendedRepo.core.repoDescription + , "homepage" .= extendedRepo.core.repoHomepage + , "language" .= (extendedRepo.core.repoLanguage <&> GH.getLanguage) + , "stargazers_count" .= extendedRepo.core.repoStargazersCount + , "open_issues_count" .= extendedRepo.core.repoOpenIssuesCount + , "commits_count" .= (extendedRepo.commitsCount & fromMaybe 0) + , "is_archived" .= extendedRepo.core.repoArchived + , "created_utc" .= (extendedRepo.core.repoCreatedAt <&> iso8601Show) + , "updated_utc" .= (extendedRepo.core.repoUpdatedAt <&> iso8601Show) + , "crawled_utc" .= now + ] + + insertRequest = + setRequestFields + airseqWriteToken + upsertRepoQuery + (KeyMap.fromList variables) + initialInsertRequest + insertResponse <- httpLbs insertRequest manager print insertResponse @@ -278,34 +280,35 @@ saveRepoInAirsequel saveStrategy extendedRepo = do deleteRepo :: Manager -> Text -> Text -> ExtendedRepo -> IO () deleteRepo manager dbEndpoint airseqWriteToken extendedRepo = do let - deleteRepoQuery :: ExtendedRepo -> Text - deleteRepoQuery extRepo = - let - repo = extRepo.core - in - [r| - mutation DeleteRepo { - delete_repos( - filter: { - github_id: { eq: <> } - } - ) { - affected_rows - returning { - owner - name - rowid - } + deleteRepoQuery :: Text + deleteRepoQuery = + [r| + mutation DeleteRepo($github_id: Int!) { + delete_repos( + filter: { + github_id: { eq: $github_id } + } + ) { + affected_rows + returning { + owner + name + rowid } } - |] - & var "github_id" (repo.repoId & GH.untagId & show) + } + |] initialDeleteRequest <- parseRequest $ T.unpack dbEndpoint + let deleteRequest = setRequestFields airseqWriteToken - (deleteRepoQuery extendedRepo) + deleteRepoQuery + ( KeyMap.fromList + ["github_id" .= (extendedRepo.core.repoId & GH.untagId)] + ) initialDeleteRequest + deleteResponse <- httpLbs deleteRequest manager print deleteResponse diff --git a/app/Main.hs b/app/Main.hs index 3e2442d..ad65501 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -39,12 +39,9 @@ import Protolude ( import Protolude qualified as P import Control.Arrow ((>>>)) -import Data.Aeson ( - eitherDecode, - encode, - object, - (.=), - ) +import Data.Aeson (Value (String), eitherDecode, encode, object, (.=)) +import Data.Aeson.KeyMap (KeyMap) +import Data.Aeson.KeyMap qualified as KeyMap import Data.List (lookup) import Data.Text qualified as T import GHC.Base (String) @@ -86,7 +83,7 @@ import Text.RawString.QQ (r) import Airsequel (saveRepoInAirsequel) import Types (ExtendedRepo (..), GqlResponse (..), SaveStrategy (..)) -import Utils (loadGitHubToken, mapMSequentially, var) +import Utils (loadGitHubToken, mapMSequentially) -- TODO: Add CLI flag to choose between OverwriteRepo and AddRepo @@ -113,7 +110,14 @@ commands = do (info upload (progDesc "Upload a single repo")) <> command "search" - (info search (progDesc "Search for and upload several repos")) + ( info + search + ( progDesc + "Search for and upload several repos.\n\ + \WARNING: If the search query returns too many repos, \ + \the result will be truncated." + ) + ) ) @@ -249,10 +253,10 @@ getGhHeaders tokenMb = execGithubGqlQuery :: Maybe Text -> Text - -> Maybe Text + -> KeyMap Value -> [ExtendedRepo] -> IO [ExtendedRepo] -execGithubGqlQuery ghTokenMb query nextCursorMb initialRepos = do +execGithubGqlQuery ghTokenMb query variables initialRepos = do manager <- newManager tlsManagerSettings initialRequest <- parseRequest $ T.unpack "https://api.github.com/graphql" @@ -265,17 +269,8 @@ execGithubGqlQuery ghTokenMb query nextCursorMb initialRepos = do RequestBodyLBS $ encode $ object - [ "query" - .= ( query - & var - "optionalAfter" - ( nextCursorMb - <&> ( \cursor -> - "after: \"" <> cursor <> "\"" - ) - & fromMaybe "" - ) - ) + [ "query" .= query + , "variables" .= variables ] } @@ -295,7 +290,7 @@ execGithubGqlQuery ghTokenMb query nextCursorMb initialRepos = do let repos :: [GH.Repo] = gqlResponse.repos - delayBetweenRequests = 1000 -- ms + delayBetweenRequests = 500 -- ms commitsCounts <- mapMSequentially delayBetweenRequests @@ -327,20 +322,29 @@ execGithubGqlQuery ghTokenMb query nextCursorMb initialRepos = do execGithubGqlQuery ghTokenMb query - (Just nextCursor) + (variables & KeyMap.insert "after" (String nextCursor)) (initialRepos <> extendedRepos) -loadAndSaveReposViaSearch :: Maybe Text -> Text -> Int -> IO [ExtendedRepo] -loadAndSaveReposViaSearch githubToken searchQuery numRepos = do +loadAndSaveReposViaSearch + :: Maybe Text + -> Text + -> Int + -> Maybe Text + -> IO [ExtendedRepo] +loadAndSaveReposViaSearch ghTokenMb searchQuery numRepos afterMb = do let gqlQUery = [r| - { + query SearchRepos( + $searchQuery: String! + $numRepos: Int! + $after: String + ){ search( - query: "<>", - type: REPOSITORY, - first: <> - <> + query: $searchQuery + type: REPOSITORY + first: $numRepos + after: $after ) { edges { node { @@ -366,13 +370,16 @@ loadAndSaveReposViaSearch githubToken searchQuery numRepos = do } } |] - & var "searchQuery" searchQuery - & var "numRepos" (show numRepos) execGithubGqlQuery - githubToken + ghTokenMb gqlQUery - Nothing + ( KeyMap.fromList + [ "searchQuery" .= searchQuery + , "numRepos" .= numRepos + , "after" .= afterMb + ] + ) [] @@ -416,7 +423,7 @@ run cliCmd = do & T.replace "\n" " " & T.strip - repos <- loadAndSaveReposViaSearch ghTokenMb searchQuery 20 + repos <- loadAndSaveReposViaSearch ghTokenMb searchQuery 20 Nothing putText $ "Found " <> show (P.length repos) <> " repos:" repos diff --git a/app/Utils.hs b/app/Utils.hs index 8125464..8cc1b62 100644 --- a/app/Utils.hs +++ b/app/Utils.hs @@ -1,11 +1,9 @@ module Utils ( - encodeToText, loadAirsWriteToken, loadDbEndpoint, loadDbId, loadGitHubToken, mapMSequentially, - var, ) where @@ -14,31 +12,22 @@ import Protolude ( Int, Maybe (..), Text, - decodeUtf8, liftIO, mapM, pure, putErrText, ($), (*), - (.), (<*), (<>), ) import Control.Concurrent (threadDelay) -import Data.Aeson (ToJSON, encode) -import Data.ByteString.Lazy (toStrict) import Data.Text qualified as T import System.Environment (lookupEnv) import System.Exit (die) -encodeToText :: (ToJSON a) => a -> Text -encodeToText = - decodeUtf8 . toStrict . encode - - lookupEnvOrDie :: Text -> IO Text lookupEnvOrDie envVarName = do envVarMb <- lookupEnv (T.unpack envVarName) @@ -82,12 +71,6 @@ loadGitHubToken = do pure $ Just $ T.pack token --- | Replaces a variable in a string with a value -var :: Text -> Text -> Text -> Text -var idName = - T.replace ("<<" <> idName <> ">>") - - mapMSequentially :: Int -> (a -> IO b) -> [a] -> IO [b] mapMSequentially delayInMs f xs = do let delayM = liftIO $ threadDelay (delayInMs * 1000)