diff --git a/README.md b/README.md index cc967d1..de4a48a 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,18 @@ Load metadata for a list of GitHub repos and store it in Airsequel. +# TODOs + +- [ ] Support specifying several search queries +- [ ] Add subcommand to load list of repos from Airsequel and update them +- [ ] Add CLI flag to choose between `OverwriteRepo` and `AddRepo` +- [ ] Store all languages for a repo + + ## Related - [GrimoireLab] - Open source tools for software development analytics. +- [SEART GitHub Search Engine] - Platform to crawl, store, and present repos. [GrimoireLab]: http://chaoss.github.io/grimoirelab/ +[SEART GitHub Search Engine]: https://github.com/seart-group/ghs diff --git a/app/Airsequel.hs b/app/Airsequel.hs index 40d5f57..cf730d0 100644 --- a/app/Airsequel.hs +++ b/app/Airsequel.hs @@ -46,8 +46,7 @@ import Data.Aeson.Types (parseEither) import Data.Text qualified as T import Data.Time (getCurrentTime) import Data.Time.Format.ISO8601 (iso8601Show) -import GitHub qualified as GH -import GitHub.Endpoints.Activity.Starring as GH (untagName) +import GHC.Base (String) import Network.HTTP.Client ( Manager, Request, @@ -65,7 +64,7 @@ import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Types (statusCode) import Text.RawString.QQ (r) -import Types (ExtendedRepo (..), SaveStrategy (..)) +import Types (GqlRes (..), Repo (..), SaveStrategy (..)) import Utils (loadAirsWriteToken, loadDbEndpoint) @@ -151,10 +150,10 @@ upsertRepoQuery = do -- | Get rowid of a repo with the specified GitHub ID -getRowid :: Manager -> Text -> Text -> ExtendedRepo -> IO (Maybe Int) -getRowid manager dbEndpoint airseqWriteToken extendedRepo = do +getRowid :: Manager -> Text -> Text -> Repo -> IO (Maybe Int) +getRowid manager dbEndpoint airseqWriteToken repo = do let - githubId = extendedRepo.core.repoId & GH.untagId + githubId = repo.githubId getRowidQuery :: Text getRowidQuery = @@ -183,12 +182,9 @@ getRowid manager dbEndpoint airseqWriteToken extendedRepo = do (putErrText $ show getRowidResponse.responseBody) let + repoSlug = repo.owner <> "/" <> repo.name msgBase = - "Repo \"" - <> (extendedRepo.core.repoOwner.simpleOwnerLogin & untagName) - <> "/" - <> (extendedRepo.core.repoName & untagName) - <> "\" is not" + "Repo \"" <> repoSlug <> "\" is not" rowidResult :: Either [P.Char] Int = ( getRowidResponse.responseBody @@ -202,7 +198,7 @@ getRowid manager dbEndpoint airseqWriteToken extendedRepo = do ) >>= ( \case [] -> Left $ T.unpack $ msgBase <> " in Airsequel yet" - [repo :: Object] -> parseEither (.: "rowid") repo + [repoObj :: Object] -> parseEither (.: "rowid") repoObj _ -> Left $ T.unpack $ @@ -215,17 +211,19 @@ getRowid manager dbEndpoint airseqWriteToken extendedRepo = do pure Nothing Right rowid -> do P.putText $ - "Repo is already in Airsequel with rowid \"" - <> show rowid - <> "\" and will be updated." + "Repo \"" + <> repoSlug + <> "\" is already in Airsequel " + <> ("(rowid " <> show rowid <> ") ") + <> "and will be updated." pure $ Just rowid {-| Insert or upsert the repo in Airsequel via a POST request executed by http-client -} -saveRepoInAirsequel :: SaveStrategy -> ExtendedRepo -> IO () -saveRepoInAirsequel saveStrategy extendedRepo = do +saveRepoInAirsequel :: SaveStrategy -> Repo -> IO () +saveRepoInAirsequel saveStrategy repo = do dbEndpoint <- loadDbEndpoint airseqWriteToken <- loadAirsWriteToken @@ -242,7 +240,7 @@ saveRepoInAirsequel saveStrategy extendedRepo = do manager dbEndpoint airseqWriteToken - extendedRepo + repo else pure Nothing initialInsertRequest <- parseRequest $ T.unpack dbEndpoint @@ -250,18 +248,18 @@ saveRepoInAirsequel saveStrategy extendedRepo = do let variables = [ "rowid" .= rowidMb - , "github_id" .= (extendedRepo.core.repoId & GH.untagId) - , "owner" .= (extendedRepo.core.repoOwner.simpleOwnerLogin & untagName) - , "name" .= (extendedRepo.core.repoName & untagName) - , "description" .= extendedRepo.core.repoDescription - , "homepage" .= extendedRepo.core.repoHomepage - , "language" .= (extendedRepo.core.repoLanguage <&> GH.getLanguage) - , "stargazers_count" .= extendedRepo.core.repoStargazersCount - , "open_issues_count" .= extendedRepo.core.repoOpenIssuesCount - , "commits_count" .= (extendedRepo.commitsCount & fromMaybe 0) - , "is_archived" .= extendedRepo.core.repoArchived - , "created_utc" .= (extendedRepo.core.repoCreatedAt <&> iso8601Show) - , "updated_utc" .= (extendedRepo.core.repoUpdatedAt <&> iso8601Show) + , "github_id" .= repo.githubId + , "owner" .= repo.owner + , "name" .= repo.name + , "description" .= repo.description + , "homepage" .= repo.homepageUrl + , "language" .= repo.primaryLanguage + , "stargazers_count" .= repo.stargazerCount + , "open_issues_count" .= repo.openIssuesCount + , "commits_count" .= (repo.commitsCount & fromMaybe 0) + , "is_archived" .= repo.isArchived + , "created_utc" .= iso8601Show repo.createdAt + , "updated_utc" .= iso8601Show repo.updatedAt , "crawled_utc" .= now ] @@ -273,12 +271,30 @@ saveRepoInAirsequel saveStrategy extendedRepo = do initialInsertRequest insertResponse <- httpLbs insertRequest manager - print insertResponse + + when (insertResponse.responseStatus.statusCode /= 200) $ do + putErrText "Http Error:" + putErrText $ show insertResponse.responseBody + + -- Check for GraphQL errors + let gqlRes :: Either String GqlRes = + insertResponse.responseBody & eitherDecode + + case gqlRes of + Left err -> do + putErrText "Error parsing GraphQL response:" + P.putStrLn err + Right GqlRes{gqlErrors} -> + case gqlErrors of + Nothing -> pure () + Just errs -> do + putErrText "GraphQL errors:" + putErrText $ show errs -- | Delete the repo from Airsequel -deleteRepo :: Manager -> Text -> Text -> ExtendedRepo -> IO () -deleteRepo manager dbEndpoint airseqWriteToken extendedRepo = do +deleteRepo :: Manager -> Text -> Text -> Repo -> IO () +deleteRepo manager dbEndpoint airseqWriteToken repo = do let deleteRepoQuery :: Text deleteRepoQuery = @@ -306,7 +322,7 @@ deleteRepo manager dbEndpoint airseqWriteToken extendedRepo = do airseqWriteToken deleteRepoQuery ( KeyMap.fromList - ["github_id" .= (extendedRepo.core.repoId & GH.untagId)] + ["github_id" .= repo.githubId] ) initialDeleteRequest diff --git a/app/Main.hs b/app/Main.hs index 890abef..0eda5ed 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -11,7 +11,6 @@ import Protolude ( Either (Left, Right), IO, Int, - Integer, Maybe (..), Text, elem, @@ -25,7 +24,6 @@ import Protolude ( pure, putErrText, putText, - readMaybe, show, when, ($), @@ -38,16 +36,12 @@ import Protolude ( ) import Protolude qualified as P -import Control.Arrow ((>>>)) import Data.Aeson (Value (String), eitherDecode, encode, object, (.=)) import Data.Aeson.KeyMap (KeyMap) import Data.Aeson.KeyMap qualified as KeyMap import Data.List (lookup) import Data.Text qualified as T import GHC.Base (String) -import GitHub qualified as GH -import GitHub.Endpoints.Activity.Starring as GH (Auth (OAuth), Repo, untagName) -import GitHub.Internal.Prelude (fromString) import Network.HTTP.Client ( RequestBody (RequestBodyLBS), Response (responseHeaders), @@ -58,7 +52,6 @@ import Network.HTTP.Client ( requestBody, requestHeaders, responseBody, - responseStatus, ) import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Link (href, parseLinkHeaderBS) @@ -70,7 +63,7 @@ import Options.Applicative ( command, execParser, fullDesc, - header, + headerDoc, helper, hsubparser, info, @@ -84,11 +77,10 @@ import Text.RawString.QQ (r) import Airsequel (saveRepoInAirsequel) import Options.Applicative.Help.Pretty (vsep) -import Types (ExtendedRepo (..), GqlResponse (..), SaveStrategy (..)) -import Utils (loadGitHubToken, mapMSequentially) +import Types (GqlRepoRes (..), Repo (..), SaveStrategy (..)) +import Utils (loadGitHubToken) --- TODO: Add CLI flag to choose between OverwriteRepo and AddRepo data CliCmd = -- | Upload a single repo Upload Text @@ -134,38 +126,19 @@ commands = do ) -formatRepo :: ExtendedRepo -> Text -formatRepo extendedRepo = - let - repo = core extendedRepo - in - "\n\n" - <> ("repo_url: " <> show (GH.repoHtmlUrl repo) <> "\n") - <> ( "description: " - <> (repo.repoDescription & fromMaybe "") - <> "\n" - ) - <> ("homepage: " <> (repo.repoHomepage & fromMaybe "") <> "\n") - <> ( "language: " - <> (repo.repoLanguage <&> GH.getLanguage & fromMaybe "") - <> "\n" - ) - <> ("stargazers_count: " <> show (GH.repoStargazersCount repo) <> "\n") - <> ( "commits_count: " - <> show (extendedRepo & commitsCount & fromMaybe 0) - <> "\n" - ) - <> ("open_issues_count: " <> show (GH.repoOpenIssuesCount repo) <> "\n") - <> ( "is_archived: " <> (repo.repoArchived & show & T.toLower) <> "\n" - ) - <> ( "created_at: " - <> (repo.repoCreatedAt <&> show & fromMaybe "") - <> "\n" - ) - <> ( "updated_at: " - <> (repo.repoUpdatedAt <&> show & fromMaybe "") - <> "\n" - ) +formatRepo :: Repo -> Text +formatRepo repo = + "\n\n" + <> ("repo_url: github.com/" <> repo.owner <> "/" <> repo.name <> "\n") + <> ("description: " <> (repo.description & fromMaybe "") <> "\n") + <> ("homepage: " <> (repo.homepageUrl & fromMaybe "") <> "\n") + <> ("language: " <> (repo.primaryLanguage & fromMaybe "") <> "\n") + <> ("stargazers_count: " <> show repo.stargazerCount <> "\n") + <> ("commits_count: " <> show (repo.commitsCount & fromMaybe 0) <> "\n") + <> ("open_issues_count: " <> show repo.openIssuesCount <> "\n") + <> ("is_archived: " <> (repo.isArchived & show & T.toLower) <> "\n") + <> ("created_at: " <> show repo.createdAt <> "\n") + <> ("updated_at: " <> show repo.updatedAt <> "\n") -- | Query @Link@ header with @rel=last@ from the request headers @@ -184,71 +157,54 @@ getLastUrl req = do pure $ href nextURI -{-| Workaround to get the number of commits for a repo -| https://stackoverflow.com/a/70610670 --} -getNumberOfCommits :: Maybe Text -> Repo -> IO (Maybe Integer) -getNumberOfCommits ghTokenMb repo = do - let repoSlug = - (repo.repoOwner.simpleOwnerLogin & untagName) - <> "/" - <> (repo.repoName & untagName) - - putText $ "⏳ Get number of commits for repo " <> repoSlug - - let apiEndpoint = - "https://api.github.com/repos/" - <> repoSlug - <> "/commits?per_page=1" - - manager <- newManager tlsManagerSettings - initialRequest <- parseRequest $ T.unpack apiEndpoint - let request = - initialRequest - { method = "HEAD" - , requestHeaders = getGhHeaders ghTokenMb - } - - response <- httpLbs request manager - - putText $ "✅ Got number of commits for repo " <> repoSlug <> ":" - putText $ show response.responseStatus - putText $ show response.responseHeaders - - getLastUrl response - <&> (show >>> T.pack >>> T.splitOn "&page=") - >>= lastMay - >>= readMaybe - & pure - - {-| Loads a single repo from GitHub, adds number of commits, | and saves it to Airsequel -} -loadAndSaveRepo :: Maybe Text -> SaveStrategy -> Text -> Text -> IO () -loadAndSaveRepo ghTokenMb saveStrategy owner name = do - let - uploadFunc = case ghTokenMb of - Nothing -> GH.github' - Just ghToken -> GH.github (OAuth (encodeUtf8 ghToken)) - - repoResult <- - uploadFunc - GH.repositoryR - (fromString $ T.unpack owner) - (fromString $ T.unpack name) - - case repoResult of - Left error -> putErrText $ "Error: " <> show error - Right repo -> do - commitsCount <- getNumberOfCommits ghTokenMb repo - let extendedRepo = - ExtendedRepo - { core = repo - , commitsCount = commitsCount +loadAndSaveRepo :: Maybe Text -> SaveStrategy -> Text -> Text -> IO [Repo] +loadAndSaveRepo ghTokenMb _TODO_saveStrategy owner name = do + let gqlQUery = + [r| + query GetSingleRepo($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + name + owner { + login + } + databaseId + stargazerCount + description + homepageUrl + primaryLanguage { + name + } + issues(states: [OPEN]) { + totalCount } - putText $ formatRepo extendedRepo - saveRepoInAirsequel saveStrategy extendedRepo + isArchived + createdAt + updatedAt + defaultBranchRef { + target { + ... on Commit { + history { + totalCount + } + } + } + } + } + } + |] + + execGithubGqlQuery + ghTokenMb + gqlQUery + ( KeyMap.fromList + [ "owner" .= owner + , "name" .= name + ] + ) + [] getGhHeaders :: (P.IsString a) => Maybe Text -> [(a, P.ByteString)] @@ -263,12 +219,7 @@ getGhHeaders tokenMb = Nothing -> [] -execGithubGqlQuery - :: Maybe Text - -> Text - -> KeyMap Value - -> [ExtendedRepo] - -> IO [ExtendedRepo] +execGithubGqlQuery :: Maybe Text -> Text -> KeyMap Value -> [Repo] -> IO [Repo] execGithubGqlQuery ghTokenMb query variables initialRepos = do manager <- newManager tlsManagerSettings @@ -289,7 +240,7 @@ execGithubGqlQuery ghTokenMb query variables initialRepos = do response <- httpLbs request manager - let gqlResult :: Either String GqlResponse = + let gqlResult :: Either String GqlRepoRes = response.responseBody & eitherDecode case gqlResult of @@ -301,43 +252,45 @@ execGithubGqlQuery ghTokenMb query variables initialRepos = do Just errors -> putErrText $ "GraphQL Errors:\n" <> show errors Nothing -> pure () - let - repos :: [GH.Repo] = gqlResponse.repos - delayBetweenRequests = 500 -- ms - commitsCounts <- - mapMSequentially - delayBetweenRequests - (getNumberOfCommits ghTokenMb) - repos - - let extendedRepos = - P.zipWith - ( \repo commitsCount -> - ExtendedRepo - { core = repo - , commitsCount - } - ) - repos - commitsCounts + let repos :: [Repo] = gqlResponse.repos - when (P.not $ P.null extendedRepos) $ do + putText $ + "✅ Received " + <> show (P.length repos) + <> " repos from GitHub" + + repos + <&> ( \repo -> + repo.owner + <> ("/" :: Text) + <> repo.name + <> (" | stars: " :: Text) + <> show repo.stargazerCount + <> (" | commits: " :: Text) + <> ( repo.commitsCount + <&> show + & fromMaybe "ERROR: Should have a commits count" + ) + ) + & mapM_ putText + + when (P.not $ P.null repos) $ do putText $ "⏳ Save " <> show (P.length repos) <> " repos to Airsequel …" -- TODO: Save all repos in one request - extendedRepos + repos & mapM_ (saveRepoInAirsequel OverwriteRepo) case gqlResponse.nextCursorMb of - Nothing -> pure $ initialRepos <> extendedRepos + Nothing -> pure $ initialRepos <> repos Just nextCursor -> do execGithubGqlQuery ghTokenMb query (variables & KeyMap.insert "after" (String nextCursor)) - (initialRepos <> extendedRepos) + (initialRepos <> repos) loadAndSaveReposViaSearch @@ -345,7 +298,7 @@ loadAndSaveReposViaSearch -> Text -> Int -> Maybe Text - -> IO [ExtendedRepo] + -> IO [Repo] loadAndSaveReposViaSearch ghTokenMb searchQuery numRepos afterMb = do let gqlQUery = [r| @@ -369,12 +322,26 @@ loadAndSaveReposViaSearch ghTokenMb searchQuery numRepos afterMb = do stargazerCount description homepageUrl + primaryLanguage { name } + # languages(first: 10) { + # totalCount + # nodes { name } + # } issues (states: [OPEN]) { totalCount } isArchived createdAt updatedAt + defaultBranchRef { + target { + ... on Commit { + history { + totalCount + } + } + } + } } } } @@ -414,27 +381,20 @@ run cliCmd = do Just owner -> case nameMb of Nothing -> putErrText "Error: Repo name is missing" - Just name -> - loadAndSaveRepo - ghTokenMb - OverwriteRepo - owner - name + Just name -> do + _ <- + loadAndSaveRepo + ghTokenMb + OverwriteRepo + owner + name + pure () Search searchQuery -> do let searchQueryNorm = searchQuery & T.replace "\n" " " & T.strip repos <- loadAndSaveReposViaSearch ghTokenMb searchQueryNorm 20 Nothing - putText $ "Found " <> show (P.length repos) <> " repos:" - repos - <&> ( \repo -> - (repo.core.repoOwner.simpleOwnerLogin & untagName) - <> ("/" :: Text) - <> (repo.core.repoName & untagName) - <> (" " :: Text) - <> show repo.commitsCount - ) - & mapM_ putText + putText $ "🏁 Total number of crawled repos: " <> show (P.length repos) pure () @@ -445,8 +405,9 @@ main = do info (commands <**> helper) ( fullDesc - <> progDesc "Upload repo metadata to Airsequel" - <> header "repos-uploader" + <> headerDoc (Just "⬆️ Repos Uploader") + <> progDesc + "Crawl repos from GitHub and upload their metadata to Airsequel" ) execParser opts >>= run diff --git a/app/Types.hs b/app/Types.hs index 863fd9f..82dedc4 100644 --- a/app/Types.hs +++ b/app/Types.hs @@ -7,91 +7,58 @@ import Protolude ( Int, Integer, Maybe (..), - Proxy (Proxy), Show, Text, mapM, pure, ($), (&), - (<&>), (>>=), ) -import Data.Aeson (FromJSON, Value, withObject, (.:), (.:?)) +import Data.Aeson (FromJSON, Object, Value, withObject, (.:), (.:?)) import Data.Aeson.Types (parseJSON) -import Data.Time (UTCTime) -import GitHub qualified as GH - - -emptyOwner :: GH.SimpleOwner -emptyOwner = - GH.SimpleOwner - { GH.simpleOwnerLogin = "" - , GH.simpleOwnerAvatarUrl = GH.URL "" - , GH.simpleOwnerId = GH.mkId (Proxy :: Proxy GH.Owner) 0 - , GH.simpleOwnerUrl = GH.URL "" - , GH.simpleOwnerType = GH.OwnerUser - } - - -emptyRepo :: GH.Repo -emptyRepo = - GH.Repo - { GH.repoOwner = emptyOwner - , GH.repoArchived = False - , GH.repoCloneUrl = Nothing - , GH.repoCreatedAt = Nothing - , GH.repoDefaultBranch = Nothing - , GH.repoDescription = Nothing - , GH.repoDisabled = False - , GH.repoForksCount = 0 - , GH.repoHasDownloads = Nothing - , GH.repoHasIssues = Nothing - , GH.repoHasPages = Nothing - , GH.repoHasProjects = Nothing - , GH.repoHasWiki = Nothing - , GH.repoHomepage = Nothing - , GH.repoHooksUrl = GH.URL "" - , GH.repoHtmlUrl = GH.URL "" - , GH.repoId = GH.mkId (Proxy :: Proxy GH.Repo) 0 - , GH.repoLanguage = Nothing - , GH.repoName = "" - , GH.repoOpenIssuesCount = 0 - , GH.repoPermissions = Nothing - , GH.repoPushedAt = Nothing - , GH.repoSize = Nothing - , GH.repoStargazersCount = 0 - , GH.repoSvnUrl = Nothing - , GH.repoUpdatedAt = Nothing - , GH.repoWatchersCount = 0 - , GH.repoPrivate = False - , GH.repoFork = Nothing - , GH.repoUrl = GH.URL "" - , GH.repoGitUrl = Nothing - , GH.repoSshUrl = Nothing - } +import Data.Time (UTCTime (UTCTime), fromGregorian, secondsToDiffTime) -{-| To make loading data from GitHub GraphQL API easier -| we also have this simpler (in comparison to GH.Repo) data type --} -data RepoObject = RepoObject - { owner :: Text +data Repo = Repo + { rowid :: Maybe Int -- Airsequel rowid + , owner :: Text , name :: Text , githubId :: Int , stargazerCount :: Int , description :: Maybe Text , homepageUrl :: Maybe Text - , issuesCount :: Int + , primaryLanguage :: Maybe Text + , openIssuesCount :: Int , isArchived :: Bool , createdAt :: UTCTime , updatedAt :: UTCTime + , commitsCount :: Maybe Integer } deriving (Show, Eq, Generic) -instance FromJSON RepoObject where +emptyRepo :: Repo +emptyRepo = + Repo + { rowid = Nothing + , owner = "" + , name = "" + , githubId = 0 + , stargazerCount = 0 + , description = Nothing + , homepageUrl = Nothing + , primaryLanguage = Nothing + , openIssuesCount = 0 + , isArchived = False + , createdAt = UTCTime (fromGregorian 1900 1 1) (secondsToDiffTime 0) + , updatedAt = UTCTime (fromGregorian 1900 1 1) (secondsToDiffTime 0) + , commitsCount = Nothing + } + + +instance FromJSON Repo where parseJSON = withObject "RepoObject" $ \o -> do owner <- o .: "owner" >>= (.: "login") name <- o .: "name" @@ -99,67 +66,73 @@ instance FromJSON RepoObject where stargazerCount <- o .: "stargazerCount" description <- o .: "description" homepageUrl <- o .: "homepageUrl" - issuesCount <- o .: "issues" >>= (.: "totalCount") + primaryLanguage <- o .: "primaryLanguage" >>= (.: "name") + openIssuesCount <- o .: "issues" >>= (.: "totalCount") isArchived <- o .: "isArchived" createdAt <- o .: "createdAt" updatedAt <- o .: "updatedAt" + commitsCount <- + o .: "defaultBranchRef" + >>= (.: "target") + >>= (.: "history") + >>= (.: "totalCount") + + pure Repo{rowid = Nothing, ..} - pure RepoObject{..} - - -repoObjectToRepo :: RepoObject -> GH.Repo -repoObjectToRepo repoObj = - emptyRepo - { GH.repoOwner = - emptyOwner - { GH.simpleOwnerLogin = - GH.mkOwnerName repoObj.owner - } - , GH.repoName = GH.mkRepoName repoObj.name - , GH.repoId = GH.mkId (Proxy :: Proxy GH.Repo) repoObj.githubId - , GH.repoHomepage = repoObj.homepageUrl - , GH.repoDescription = repoObj.description - , GH.repoStargazersCount = repoObj.stargazerCount - , GH.repoOpenIssuesCount = repoObj.issuesCount - , GH.repoArchived = repoObj.isArchived - , GH.repoCreatedAt = Just repoObj.createdAt - , GH.repoUpdatedAt = Just repoObj.updatedAt - } + +-- | Generic GraphQL response +data GqlRes = GqlRes + { gqlData :: Maybe Value + , gqlErrors :: Maybe [Object] + } + deriving (Show, Eq, Generic) -data GqlResponse = GqlResponse - { repos :: [GH.Repo] +instance FromJSON GqlRes where + parseJSON = withObject "GqlRes" $ \o -> do + gqlData <- o .:? "data" + gqlErrors <- o .:? "errors" + pure GqlRes{..} + + +data GqlRepoRes = GqlRepoRes + { repos :: [Repo] , errorsMb :: Maybe Value , nextCursorMb :: Maybe Text } deriving (Show, Eq, Generic) -instance FromJSON GqlResponse where - parseJSON = withObject "GqlResponse" $ \o -> do +instance FromJSON GqlRepoRes where + parseJSON = withObject "GqlRepoRes" $ \o -> do data_ <- o .: "data" errorsMb <- o .:? "errors" - search <- data_ .: "search" - edges <- search .: "edges" - repos :: [RepoObject] <- edges & mapM (.: "node") - - pageInfo <- search .: "pageInfo" - nextCursorMb <- pageInfo .:? "endCursor" - - pure - GqlResponse - { repos = repos <&> repoObjectToRepo - , errorsMb - , nextCursorMb - } + searchMb <- data_ .:? "search" + + case searchMb of + Nothing -> do + repository <- data_ .: "repository" + repo <- parseJSON repository + pure + GqlRepoRes + { repos = [repo] + , errorsMb + , nextCursorMb = Nothing + } + Just search -> do + edges <- search .: "edges" + repos :: [Repo] <- edges & mapM (.: "node") + + pageInfo <- search .: "pageInfo" + nextCursorMb <- pageInfo .:? "endCursor" + + pure + GqlRepoRes + { repos = repos + , errorsMb + , nextCursorMb + } data SaveStrategy = OverwriteRepo | AddRepo deriving (Show, Eq) - - -data ExtendedRepo = ExtendedRepo - { core :: GH.Repo - , commitsCount :: Maybe Integer - } - deriving (Show, Eq) diff --git a/app/Utils.hs b/app/Utils.hs index 8cc1b62..dfc0cf0 100644 --- a/app/Utils.hs +++ b/app/Utils.hs @@ -3,26 +3,19 @@ module Utils ( loadDbEndpoint, loadDbId, loadGitHubToken, - mapMSequentially, ) where import Protolude ( IO, - Int, Maybe (..), Text, - liftIO, - mapM, pure, putErrText, ($), - (*), - (<*), (<>), ) -import Control.Concurrent (threadDelay) import Data.Text qualified as T import System.Environment (lookupEnv) import System.Exit (die) @@ -69,9 +62,3 @@ loadGitHubToken = do pure Nothing Just token -> pure $ Just $ T.pack token - - -mapMSequentially :: Int -> (a -> IO b) -> [a] -> IO [b] -mapMSequentially delayInMs f xs = do - let delayM = liftIO $ threadDelay (delayInMs * 1000) - mapM (\x -> f x <* delayM) xs diff --git a/package.yaml b/package.yaml index 72a6271..d25b866 100644 --- a/package.yaml +++ b/package.yaml @@ -18,6 +18,8 @@ dependencies: - aeson - base - bytestring + - http-client + - http-client-tls - http-link-header - http-types - network-uri @@ -46,17 +48,9 @@ ghc-options: library: language: GHC2021 source-dirs: source - dependencies: - - http-client-tls - - http-client - - github executables: repos-uploader: language: GHC2021 source-dirs: app main: Main.hs - dependencies: - - http-client-tls - - http-client - - github diff --git a/repos-uploader.cabal b/repos-uploader.cabal index 7b747b8..d7785ac 100644 --- a/repos-uploader.cabal +++ b/repos-uploader.cabal @@ -4,7 +4,7 @@ cabal-version: 2.2 -- -- see: https://github.com/sol/hpack -- --- hash: 63fc3e8744bf308b5a7b0c2268b75bb440d310bd2dc36e72791d893e12b745c3 +-- hash: 0862c588be00e327ee1b6cc1fac5f380af9fcf1ab0294c48aa90748353d89655 name: repos-uploader version: 0.0.0.0 @@ -42,7 +42,6 @@ library aeson , base , bytestring - , github , http-client , http-client-tls , http-link-header @@ -78,7 +77,6 @@ executable repos-uploader aeson , base , bytestring - , github , http-client , http-client-tls , http-link-header