Show GraphQL errors if rowids can't be loaded from Airsequel

Airsequel · Feb 20, 2024 · db191e5 · db191e5
1 parent 9ed384e
commit db191e5
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 49 deletions.
diff --git a/README.md b/README.md
@@ -39,13 +39,13 @@ Available commands:
 
 ## TODOs
 
-- [ ] Add a column repo.exclusion_reason
-      If `!= NULL` repo should be excluded from further processing
-      E.g. for forks, mirrors, etc.
-- [ ] Add subcommand to load list of repos from Airsequel and update them
-- [ ] Add CLI flag to choose between `OverwriteRepo` and `AddRepo`
-- [ ] Store all languages for a repo
-- [ ] Store if account is a person or an organization
+- Add column `is_private` and only crawl private repos if `--private` is passed
+- Add subcommand to load list of repos from Airsequel and update them
+- Move `bin-calculation.py` to Airsequel
+- Store if account is a person or an organization
+- Store all languages for a repo
+- Repos created per week chart
+- Add CLI flag to choose between `OverwriteRepo` and `AddRepo`
 
 
 ## Related

diff --git a/airput.cabal b/airput.cabal
@@ -37,6 +37,7 @@ library
   ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-orphans
   build-depends:
       aeson
+    , aeson-pretty
     , base
     , bytestring
     , directory
@@ -76,6 +77,7 @@ executable airput
   ghc-options: -Wall -Wcompat -Wincomplete-record-updates -Wincomplete-uni-patterns -Wredundant-constraints -fno-warn-orphans
   build-depends:
       aeson
+    , aeson-pretty
     , base
     , bytestring
     , directory

diff --git a/app/Airsequel.hs b/app/Airsequel.hs
@@ -32,21 +32,21 @@ import Protolude (
   (<&>),
   (<>),
   (==),
-  (>>=),
  )
 import Protolude qualified as P
 
 import Control.Arrow ((>>>))
 import Data.Aeson (
   Object,
+  Value (Object),
   eitherDecode,
   encode,
   object,
   (.:),
   (.=),
  )
 import Data.Aeson.KeyMap qualified as KeyMap
-import Data.Aeson.Types (parseEither)
+import Data.Aeson.Types (Parser, parseEither)
 import Data.Text qualified as T
 import Data.Time (getCurrentTime)
 import Data.Time.Format.ISO8601 (iso8601Show)
@@ -68,6 +68,7 @@ import Network.HTTP.Client.TLS (tlsManagerSettings)
 import Network.HTTP.Types (statusCode)
 import Text.RawString.QQ (r)
 
+import Data.Aeson.Encode.Pretty (encodePretty)
 import Types (GqlRes (..), Repo (..), SaveStrategy (..))
 import Utils (loadAirsWriteToken, loadDbEndpoint)
 
@@ -153,46 +154,69 @@ loadRowids manager dbEndpoint airseqWriteToken repos = do
     (getReposWithRowidResponse.responseStatus.statusCode /= 200)
     (putErrText $ show getReposWithRowidResponse.responseBody)
 
-  let
-    ghIdsWithRowid
-      :: Either [P.Char] [(Integer {- githubId -}, Integer {- rowid -})] =
-        (getReposWithRowidResponse.responseBody & eitherDecode)
-          >>= ( \gqlRes ->
-                  P.flip parseEither gqlRes $ \gqlResObj -> do
-                    gqlData <- gqlResObj .: "data"
-                    gqlData .: "repos"
-              )
-            <&> ( \(reposWithRowid :: [Repo]) ->
-                    reposWithRowid
-                      & filter (\repoWithRowid -> isJust repoWithRowid.rowid)
-                      <&> ( \repoWithRowid ->
-                              ( repoWithRowid.githubId
-                              , repoWithRowid.rowid & fromMaybe 0
-                              )
-                          )
-                )
+  -- Check for GraphQL errors
+  let gqlRes :: Either String GqlRes =
+        getReposWithRowidResponse.responseBody
+          & eitherDecode
 
-  case ghIdsWithRowid of
+  case gqlRes of
     Left err -> do
+      putErrText "Error parsing GraphQL response:"
       putErrLn err
       pure repos
-    Right rowids -> do
-      P.putText $
-        "\nℹ️ "
-          <> show @Int (P.length rowids)
-          <> " of "
-          <> show @Int (P.length repos)
-          <> " repos already exist in Airsequel"
-
-      pure $
-        repos <&> \repo ->
-          repo
-            { rowid =
-                rowids
-                  & filter (\(ghId, _) -> ghId == repo.githubId)
-                  & P.head
-                  <&> P.snd
-            }
+    Right GqlRes{gqlErrors, gqlData} ->
+      case gqlErrors of
+        Just errs -> do
+          putErrText "GraphQL errors:"
+          errs
+            <&> encodePretty
+            & P.mapM_ P.putLByteString
+          pure repos
+        Nothing -> do
+          let
+            repoParser :: Maybe Value -> Parser [Repo]
+            repoParser = \case
+              Just (Object obj) -> obj .: "repos"
+              _ -> P.mempty
+
+            -- \| … [(githubId, rowid)]
+            ghIdsWithRowid :: Either [P.Char] [(Integer, Integer)] =
+              gqlData
+                & parseEither repoParser
+                <&> ( \(reposWithRowid :: [Repo]) ->
+                        reposWithRowid
+                          & filter
+                            ( \repoWithRowid ->
+                                isJust repoWithRowid.rowid
+                            )
+                          <&> ( \repoWithRowid ->
+                                  ( repoWithRowid.githubId
+                                  , repoWithRowid.rowid & fromMaybe 0
+                                  )
+                              )
+                    )
+
+          case ghIdsWithRowid of
+            Left err -> do
+              putErrLn err
+              pure repos
+            Right rowids -> do
+              P.putText $
+                "\nℹ️ "
+                  <> show @Int (P.length rowids)
+                  <> " of "
+                  <> show @Int (P.length repos)
+                  <> " repos already exist in Airsequel"
+
+              pure $
+                repos <&> \repo ->
+                  repo
+                    { rowid =
+                        rowids
+                          & filter (\(ghId, _) -> ghId == repo.githubId)
+                          & P.head
+                          <&> P.snd
+                    }
 
 
 {-| Insert or upsert repos in Airsequel

diff --git a/app/Main.hs b/app/Main.hs
@@ -146,7 +146,9 @@ commands = do
           "github-upload"
           ( info
               githubUpload
-              (progDesc "Upload metadata for a single GitHub repo")
+              ( progDesc
+                  "Upload metadata for a single GitHub repo \"owner/name\""
+              )
           )
         <> command
           "github-search"
@@ -283,12 +285,17 @@ execGithubGqlQuery ghTokenMb query variables initialRepos = do
               <> "⚠️ The search returns more than 1000 repos.\n"
               <> "⚠️ Not all repos will be crawled.\n"
 
-      let repos :: [Repo] = gqlResponse.repos
+      let
+        repos :: [Repo] = gqlResponse.repos
+        numRepos = P.length repos
+        pluralize word = if numRepos > 1 then word <> "s" else word
 
       putText $
         "\n✅ Received "
-          <> show @Int (P.length repos)
-          <> " repos from GitHub"
+          <> show @Int numRepos
+          <> " "
+          <> pluralize "repo"
+          <> " from GitHub"
 
       repos
         <&> ( \repo ->

diff --git a/package.yaml b/package.yaml
@@ -14,6 +14,7 @@ extra-source-files:
 
 dependencies:
   - aeson
+  - aeson-pretty
   - base
   - bytestring
   - directory