From db191e59917366e20389f9784cc7f5755c37481d Mon Sep 17 00:00:00 2001 From: Adrian Sieber Date: Tue, 20 Feb 2024 17:07:22 +0000 Subject: [PATCH] Show GraphQL errors if rowids can't be loaded from Airsequel --- README.md | 14 +++---- airput.cabal | 2 + app/Airsequel.hs | 100 +++++++++++++++++++++++++++++------------------ app/Main.hs | 15 +++++-- package.yaml | 1 + 5 files changed, 83 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 3e494ee..d4fc9b9 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ Available commands: ## TODOs -- [ ] Add a column repo.exclusion_reason - If `!= NULL` repo should be excluded from further processing - E.g. for forks, mirrors, etc. -- [ ] Add subcommand to load list of repos from Airsequel and update them -- [ ] Add CLI flag to choose between `OverwriteRepo` and `AddRepo` -- [ ] Store all languages for a repo -- [ ] Store if account is a person or an organization +- Add column `is_private` and only crawl private repos if `--private` is passed +- Add subcommand to load list of repos from Airsequel and update them +- Move `bin-calculation.py` to Airsequel +- Store if account is a person or an organization +- Store all languages for a repo +- Repos created per week chart +- Add CLI flag to choose between `OverwriteRepo` and `AddRepo` ## Related diff --git a/airput.cabal b/airput.cabal index 1d0876d..73a6f56 100644 --- a/airput.cabal +++ b/airput.cabal @@ -37,6 +37,7 @@ library ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-orphans build-depends: aeson + , aeson-pretty , base , bytestring , directory @@ -76,6 +77,7 @@ executable airput ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-orphans build-depends: aeson + , aeson-pretty , base , bytestring , directory diff --git a/app/Airsequel.hs b/app/Airsequel.hs index 8166c5e..b6297ab 100644 --- a/app/Airsequel.hs +++ b/app/Airsequel.hs @@ -32,13 +32,13 @@ import Protolude ( (<&>), (<>), (==), - (>>=), ) import Protolude qualified as P import Control.Arrow ((>>>)) import Data.Aeson ( Object, + Value (Object), eitherDecode, encode, object, @@ -46,7 +46,7 @@ import Data.Aeson ( (.=), ) import Data.Aeson.KeyMap qualified as KeyMap -import Data.Aeson.Types (parseEither) +import Data.Aeson.Types (Parser, parseEither) import Data.Text qualified as T import Data.Time (getCurrentTime) import Data.Time.Format.ISO8601 (iso8601Show) @@ -68,6 +68,7 @@ import Network.HTTP.Client.TLS (tlsManagerSettings) import Network.HTTP.Types (statusCode) import Text.RawString.QQ (r) +import Data.Aeson.Encode.Pretty (encodePretty) import Types (GqlRes (..), Repo (..), SaveStrategy (..)) import Utils (loadAirsWriteToken, loadDbEndpoint) @@ -153,46 +154,69 @@ loadRowids manager dbEndpoint airseqWriteToken repos = do (getReposWithRowidResponse.responseStatus.statusCode /= 200) (putErrText $ show getReposWithRowidResponse.responseBody) - let - ghIdsWithRowid - :: Either [P.Char] [(Integer {- githubId -}, Integer {- rowid -})] = - (getReposWithRowidResponse.responseBody & eitherDecode) - >>= ( \gqlRes -> - P.flip parseEither gqlRes $ \gqlResObj -> do - gqlData <- gqlResObj .: "data" - gqlData .: "repos" - ) - <&> ( \(reposWithRowid :: [Repo]) -> - reposWithRowid - & filter (\repoWithRowid -> isJust repoWithRowid.rowid) - <&> ( \repoWithRowid -> - ( repoWithRowid.githubId - , repoWithRowid.rowid & fromMaybe 0 - ) - ) - ) + -- Check for GraphQL errors + let gqlRes :: Either String GqlRes = + getReposWithRowidResponse.responseBody + & eitherDecode - case ghIdsWithRowid of + case gqlRes of Left err -> do + putErrText "Error parsing GraphQL response:" putErrLn err pure repos - Right rowids -> do - P.putText $ - "\nℹ️ " - <> show @Int (P.length rowids) - <> " of " - <> show @Int (P.length repos) - <> " repos already exist in Airsequel" - - pure $ - repos <&> \repo -> - repo - { rowid = - rowids - & filter (\(ghId, _) -> ghId == repo.githubId) - & P.head - <&> P.snd - } + Right GqlRes{gqlErrors, gqlData} -> + case gqlErrors of + Just errs -> do + putErrText "GraphQL errors:" + errs + <&> encodePretty + & P.mapM_ P.putLByteString + pure repos + Nothing -> do + let + repoParser :: Maybe Value -> Parser [Repo] + repoParser = \case + Just (Object obj) -> obj .: "repos" + _ -> P.mempty + + -- \| … [(githubId, rowid)] + ghIdsWithRowid :: Either [P.Char] [(Integer, Integer)] = + gqlData + & parseEither repoParser + <&> ( \(reposWithRowid :: [Repo]) -> + reposWithRowid + & filter + ( \repoWithRowid -> + isJust repoWithRowid.rowid + ) + <&> ( \repoWithRowid -> + ( repoWithRowid.githubId + , repoWithRowid.rowid & fromMaybe 0 + ) + ) + ) + + case ghIdsWithRowid of + Left err -> do + putErrLn err + pure repos + Right rowids -> do + P.putText $ + "\nℹ️ " + <> show @Int (P.length rowids) + <> " of " + <> show @Int (P.length repos) + <> " repos already exist in Airsequel" + + pure $ + repos <&> \repo -> + repo + { rowid = + rowids + & filter (\(ghId, _) -> ghId == repo.githubId) + & P.head + <&> P.snd + } {-| Insert or upsert repos in Airsequel diff --git a/app/Main.hs b/app/Main.hs index 8b9dc6a..965e0e6 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -146,7 +146,9 @@ commands = do "github-upload" ( info githubUpload - (progDesc "Upload metadata for a single GitHub repo") + ( progDesc + "Upload metadata for a single GitHub repo \"owner/name\"" + ) ) <> command "github-search" @@ -283,12 +285,17 @@ execGithubGqlQuery ghTokenMb query variables initialRepos = do <> "⚠️ The search returns more than 1000 repos.\n" <> "⚠️ Not all repos will be crawled.\n" - let repos :: [Repo] = gqlResponse.repos + let + repos :: [Repo] = gqlResponse.repos + numRepos = P.length repos + pluralize word = if numRepos > 1 then word <> "s" else word putText $ "\n✅ Received " - <> show @Int (P.length repos) - <> " repos from GitHub" + <> show @Int numRepos + <> " " + <> pluralize "repo" + <> " from GitHub" repos <&> ( \repo -> diff --git a/package.yaml b/package.yaml index d1408d1..af4b798 100644 --- a/package.yaml +++ b/package.yaml @@ -14,6 +14,7 @@ extra-source-files: dependencies: - aeson + - aeson-pretty - base - bytestring - directory