Support multiple --issue and --pull-request options

Refs #48

README.md

@@ -61,19 +61,19 @@ You can point to a different location of `auth.json` using `-a`:
 
     $ github-to-sqlite issues github.db simonw/datasette -a /path/to/auth.json
 
-You can use the `--issue` option to only load just one specific issue:
+You can use the `--issue` option one or more times to load specific issues:
 
     $ github-to-sqlite issues github.db simonw/datasette --issue=1
 
-## Fetching pull-requests for a repository
+## Fetching pull requests for a repository
 
-While pull-requests are a type of issue, you will get more information on pull-requests by pulling them separately. For example, whether a pull-request has been merged and when.
+While pull requests are a type of issue, you will get more information on pull requests by pulling them separately. For example, whether a pull request has been merged and when.
 
-Following the API of issues, the `pull-requests` command retrieves all of the pull-requests belonging to a specified repository.
+Following the API of issues, the `pull-requests` command retrieves all of the pull requests belonging to a specified repository.
 
     $ github-to-sqlite pull-requests github.db simonw/datasette
 
-You can use the `--pull-request` option to only load just one specific pull-request:
+You can use the `--pull-request` option one or more times to load specific pull request:
 
     $ github-to-sqlite pull-requests github.db simonw/datasette --pull-request=81
 

github_to_sqlite/cli.py

@@ -42,8 +42,14 @@ def auth(auth):
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
     required=True,
 )
-@click.argument("repo", required=False)
-@click.option("--issue", help="Just pull this issue number")
+@click.argument("repo")
+@click.option(
+    "--issue",
+    "issue_ids",
+    help="Just pull these issue numbers",
+    type=int,
+    multiple=True,
+)
 @click.option(
     "-a",
     "--auth",
@@ -56,28 +62,36 @@ def auth(auth):
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
     help="Load issues JSON from this file instead of the API",
 )
-def issues(db_path, repo, issue, auth, load):
+def issues(db_path, repo, issue_ids, auth, load):
     "Save issues for a specified repository, e.g. simonw/datasette"
     db = sqlite_utils.Database(db_path)
     token = load_token(auth)
     repo_full = utils.fetch_repo(repo, token)
+    utils.save_repo(db, repo_full)
     if load:
         issues = json.load(open(load))
     else:
-        issues = utils.fetch_issues(repo, token, issue)
+        issues = utils.fetch_issues(repo, token, issue_ids)
 
     issues = list(issues)
     utils.save_issues(db, issues, repo_full)
     utils.ensure_db_shape(db)
 
+
 @cli.command(name="pull-requests")
 @click.argument(
     "db_path",
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
     required=True,
 )
 @click.argument("repo", required=False)
-@click.option("--pull-request", help="Just pull this pull-request number")
+@click.option(
+    "--pull-request",
+    "pull_request_ids",
+    help="Just pull these pull-request numbers",
+    type=int,
+    multiple=True,
+)
 @click.option(
     "-a",
     "--auth",
@@ -90,15 +104,16 @@ def issues(db_path, repo, issue, auth, load):
     type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True),
     help="Load pull-requests JSON from this file instead of the API",
 )
-def pull_requests(db_path, repo, pull_request, auth, load):
+def pull_requests(db_path, repo, pull_request_ids, auth, load):
     "Save pull_requests for a specified repository, e.g. simonw/datasette"
     db = sqlite_utils.Database(db_path)
     token = load_token(auth)
     repo_full = utils.fetch_repo(repo, token)
+    utils.save_repo(db, repo_full)
     if load:
         pull_requests = json.load(open(load))
     else:
-        pull_requests = utils.fetch_pull_requests(repo, token, pull_request)
+        pull_requests = utils.fetch_pull_requests(repo, token, pull_request_ids)
 
     pull_requests = list(pull_requests)
     utils.save_pull_requests(db, pull_requests, repo_full)

github_to_sqlite/utils.py

@@ -178,7 +178,9 @@ def save_pull_requests(db, pull_requests, repo):
         pull_request["base"] = pull_request["base"]["sha"]
         # Extract milestone
         if pull_request["milestone"]:
-            pull_request["milestone"] = save_milestone(db, pull_request["milestone"], repo["id"])
+            pull_request["milestone"] = save_milestone(
+                db, pull_request["milestone"], repo["id"]
+            )
         # For the moment we ignore the assignees=[] array but we DO turn assignee
         # singular into a foreign key reference
         pull_request.pop("assignees", None)
@@ -329,21 +331,30 @@ def save_license(db, license):
     return db["licenses"].insert(license, pk="key", replace=True).last_pk
 
 
-def fetch_issues(repo, token=None, issue=None):
+def fetch_issues(repo, token=None, issue_ids=None):
     headers = make_headers(token)
-    if issue is not None:
-        url = "https://api.github.com/repos/{}/issues/{}".format(repo, issue)
-        yield from [requests.get(url).json()]
+    if issue_ids is not None:
+        for issue_id in issue_ids:
+            url = "https://api.github.com/repos/{}/issues/{}".format(repo, issue_id)
+            response = requests.get(url)
+            response.raise_for_status()
+            yield response.json()
     else:
         url = "https://api.github.com/repos/{}/issues?state=all&filter=all".format(repo)
         for issues in paginate(url, headers):
             yield from issues
 
-def fetch_pull_requests(repo, token=None, pull_request=None):
+
+def fetch_pull_requests(repo, token=None, pull_request_ids=None):
     headers = make_headers(token)
-    if pull_request is not None:
-        url = "https://api.github.com/repos/{}/pulls/{}".format(repo, pull_request)
-        yield from [requests.get(url).json()]
+    if pull_request_ids is not None:
+        for pull_request_id in pull_request_ids:
+            url = "https://api.github.com/repos/{}/pulls/{}".format(
+                repo, pull_request_id
+            )
+            response = requests.get(url)
+            response.raise_for_status()
+            yield response.json()
     else:
         url = "https://api.github.com/repos/{}/pulls?state=all&filter=all".format(repo)
         for pull_requests in paginate(url, headers):