diff --git a/README.md b/README.md index 463509a..ae5caa0 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,39 @@ CLI tool for populating Airsequel with data. Includes a crawler for metadata of GitHub repos. -# TODOs +## Usage + +```txt +ā¬†ļø Airput ā¬†ļø + +Usage: airput COMMAND + + CLI tool for populating Airsequel with data. + +Available options: + -h,--help Show this help text + +Available commands: + github-upload Upload metadata for a single GitHub repo + github-search Search for GitHub repos and upload their metadata. + + If several search queries are provided, they will be + executed one after the other. + + WARNING: If a search returns more than 1000 repos, + the results will be truncated. + + Good search options are: + - language:haskell + - stars:>=10 + - stars:10..50 + - created:2023-10 + - archived:true +``` + + +## TODOs -- [ ] Support specifying several search queries - [ ] Add subcommand to load list of repos from Airsequel and update them - [ ] Add CLI flag to choose between `OverwriteRepo` and `AddRepo` - [ ] Store all languages for a repo diff --git a/app/Main.hs b/app/Main.hs index f16cb4b..fd4792f 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -20,6 +20,7 @@ import Protolude ( fromMaybe, headMay, lastMay, + many, mapM_, mempty, pure, @@ -78,6 +79,7 @@ import Options.Applicative ( import Text.RawString.QQ (r) import Airsequel (saveReposInAirsequel) +import Control.Arrow ((>>>)) import Options.Applicative.Help.Pretty (vsep) import Types (GqlRepoRes (..), Repo (..), SaveStrategy (..)) import Utils (loadGitHubToken) @@ -87,7 +89,7 @@ data CliCmd = -- | Upload metadata for a single GitHub repo GithubUpload Text | -- | Search for GitHub repos and upload their metadata - GithubSearch Text + GithubSearch [Text] commands :: Parser CliCmd @@ -97,7 +99,7 @@ commands = do githubUpload = GithubUpload <$> argument str (metavar "REPO_SLUG") githubSearch :: Parser CliCmd - githubSearch = GithubSearch <$> argument str (metavar "SEARCH_QUERY") + githubSearch = GithubSearch <$> many (argument str (metavar "SEARCH_QUERY")) hsubparser ( mempty @@ -116,7 +118,10 @@ commands = do vsep [ "Search for GitHub repos and upload their metadata." , "" - , "WARNING: If the search returns more than 1000 repos," + , "If several search queries are provided, they will be" + , " executed one after the other." + , "" + , "WARNING: If a search returns more than 1000 repos," , " the results will be truncated." , "" , "Good search options are:" @@ -388,17 +393,25 @@ run cliCmd = do owner name pure () - GithubSearch searchQuery -> do - let searchQueryNorm = searchQuery & T.replace "\n" " " & T.strip + GithubSearch searchQueries -> do + let searchQueriesNorm = searchQueries <&> (T.replace "\n" " " >>> T.strip) - repos <- loadAndSaveReposViaSearch ghTokenMb searchQueryNorm 20 Nothing + allRepos <- P.forM searchQueriesNorm $ \searchQueryNorm -> do + repos <- loadAndSaveReposViaSearch ghTokenMb searchQueryNorm 50 Nothing - putText $ - "\nšŸ Crawled " - <> show @Int (P.length repos) - <> " for search query šŸ\n" + putText $ + "\nšŸ Crawled " + <> show @Int (P.length repos) + <> " repos with search query:\n" + <> searchQueryNorm + <> "\n" + + pure repos - pure () + putText $ + "\nšŸšŸšŸ Crawled " + <> show @Int (P.length $ P.concat allRepos) + <> " repos in total šŸšŸšŸ\n" main :: IO ()