GitHub Search: Show total count of found repos, show warning if too big

Airsequel · Jan 8, 2024 · dae06a7 · dae06a7
1 parent b63a05e
commit dae06a7
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 9 deletions.
diff --git a/app/Airsequel.hs b/app/Airsequel.hs
@@ -178,7 +178,8 @@ loadRowids manager dbEndpoint airseqWriteToken repos = do
       pure repos
     Right rowids -> do
       P.putStrLn $
-        showInt (P.length rowids) " of "
+        "\nℹ️ "
+          <> showInt (P.length rowids) " of "
           <> showInt (P.length repos) " repos already exist in Airsequel"
 
       pure $
@@ -197,7 +198,7 @@ via a POST request executed by http-client
 -}
 saveReposInAirsequel :: SaveStrategy -> [Repo] -> IO ()
 saveReposInAirsequel saveStrategy repos = do
-  P.putText $ "⏳ Saving " <> show (P.length repos) <> " repos in Airsequel …"
+  P.putText $ "\n⏳ Saving " <> show (P.length repos) <> " repos in Airsequel …"
 
   dbEndpoint <- loadDbEndpoint
   airseqWriteToken <- loadAirsWriteToken

diff --git a/app/Main.hs b/app/Main.hs
@@ -33,6 +33,7 @@ import Protolude (
   (<$>),
   (<&>),
   (<>),
+  (>),
   (>>=),
  )
 import Protolude qualified as P
@@ -261,16 +262,28 @@ execGithubGqlQuery ghTokenMb query variables initialRepos = do
         Just errors -> putErrText $ "GraphQL Errors:\n" <> show errors
         Nothing -> pure ()
 
+      when (P.null initialRepos {- First call -}) $ do
+        putStrLn $
+          "\n📲 Total number of repos: "
+            <> showInt gqlResponse.repositoryCount ""
+
+        when (gqlResponse.repositoryCount > 1000) $ do
+          putText $
+            "\n⚠️ WARNING\n"
+              <> "⚠️ The search returns more than 1000 repos.\n"
+              <> "⚠️ Not all repos will be crawled.\n"
+
       let repos :: [Repo] = gqlResponse.repos
 
       putStrLn $
-        "✅ Received "
+        "\n✅ Received "
           <> showInt (P.length repos) " repos "
           <> "from GitHub"
 
       repos
         <&> ( \repo ->
-                (repo.owner & fromMaybe "")
+                T.replicate 4 " "
+                  <> (repo.owner & fromMaybe "")
                   <> ("/" :: Text)
                   <> (repo.name & fromMaybe "")
                   <> (" | stars: " :: Text)
@@ -316,6 +329,7 @@ loadAndSaveReposViaSearch ghTokenMb searchQuery numRepos afterMb = do
               first: $numRepos
               after: $after
             ) {
+              repositoryCount
               edges {
                 node {
                   ... on Repository {
@@ -398,8 +412,8 @@ run cliCmd = do
       repos <- loadAndSaveReposViaSearch ghTokenMb searchQueryNorm 20 Nothing
 
       putStrLn $
-        "🏁 Total number of crawled repos: "
-          <> showInt (P.length repos) ""
+        "\n🏁🏁🏁 Crawled "
+          <> showInt (P.length repos) " repos in total 🏁🏁🏁\n"
 
       pure ()
 

diff --git a/app/Types.hs b/app/Types.hs
@@ -117,7 +117,8 @@ instance FromJSON GqlRes where
 
 
 data GqlRepoRes = GqlRepoRes
-  { repos :: [Repo]
+  { repositoryCount :: Integer
+  , repos :: [Repo]
   , errorsMb :: Maybe Value
   , nextCursorMb :: Maybe Text
   }
@@ -136,11 +137,13 @@ instance FromJSON GqlRepoRes where
         repo <- parseJSON repository
         pure
           GqlRepoRes
-            { repos = [repo]
+            { repositoryCount = 1
+            , repos = [repo]
             , errorsMb
             , nextCursorMb = Nothing
             }
       Just search -> do
+        repositoryCount <- search .: "repositoryCount"
         edges <- search .: "edges"
         repos :: [Repo] <- edges & mapM (.: "node")
 
@@ -149,7 +152,8 @@ instance FromJSON GqlRepoRes where
 
         pure
           GqlRepoRes
-            { repos = repos
+            { repositoryCount
+            , repos
             , errorsMb
             , nextCursorMb
             }