diff --git a/README.md b/README.md index 6356076..a2cabda 100644 --- a/README.md +++ b/README.md @@ -166,3 +166,23 @@ If you add the `--fetch` option the command will also fetch the binary content o [########----------------------------] 397/1799 22% 00:03:43 You can then use the [datasette-render-images](https://github.com/simonw/datasette-render-images) plugin to browse them visually. + +## Making authenticated API calls + +The `github-to-sqlite get` command provides a convenient shortcut for making authenticated calls to the API. Once you have created your `auth.json` file (or set a `GITHUB_TOKEN` environment variable) you can use it like this: + + $ github-to-sqlite get https://api.github.com/gists + +This will make an authenticated call to the URL you provide and pretty-print the resulting JSON to the console. + +You can ommit the `https://api.github.com/` prefix, for example: + + $ github-to-sqlite get /gists + +Many GitHub APIs are [paginated using the HTTP Link header](https://docs.github.com/en/rest/guides/traversing-with-pagination). You can follow this pagination and output a list of all of the resulting items using `--paginate`: + + $ github-to-sqlite get /users/simonw/repos --paginate + +You can outline newline-delimited JSON for each item using `--nl`. This can be useful for streaming items into another tool. + + $ github-to-sqlite get /users/simonw/repos --nl diff --git a/github_to_sqlite/cli.py b/github_to_sqlite/cli.py index f7b02e8..a335f2c 100644 --- a/github_to_sqlite/cli.py +++ b/github_to_sqlite/cli.py @@ -1,6 +1,7 @@ import click import datetime import pathlib +import textwrap import os import sqlite_utils import time @@ -442,6 +443,52 @@ def emojis(db_path, auth, fetch): table.update(emoji["name"], {"image": utils.fetch_image(emoji["url"])}) +@cli.command() +@click.argument("url", type=str) +@click.option( + "-a", + "--auth", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + default="auth.json", + help="Path to auth.json token file", +) +@click.option( + "--paginate", + is_flag=True, + help="Paginate through all results", +) +@click.option( + "--nl", + is_flag=True, + help="Output newline-delimited JSON", +) +def get(url, auth, paginate, nl): + "Save repos owened by the specified (or authenticated) username or organization" + token = load_token(auth) + if paginate or nl: + first = True + while url: + response = utils.get(url, token) + items = response.json() + if first and not nl: + click.echo("[") + for item in items: + if not first and not nl: + click.echo(",") + first = False + if not nl: + to_dump = json.dumps(item, indent=4) + click.echo(textwrap.indent(to_dump, " "), nl=False) + else: + click.echo(json.dumps(item)) + url = response.links.get("next", {}).get("url") + if not nl: + click.echo("\n]") + else: + response = utils.get(url, token) + click.echo(json.dumps(response.json(), indent=4)) + + def load_token(auth): try: token = json.load(open(auth))["github_personal_token"] diff --git a/github_to_sqlite/utils.py b/github_to_sqlite/utils.py index e0783de..377cbd2 100644 --- a/github_to_sqlite/utils.py +++ b/github_to_sqlite/utils.py @@ -649,3 +649,12 @@ def fetch_emojis(token=None): def fetch_image(url): return requests.get(url).content + + +def get(url, token=None): + headers = make_headers(token) + if url.startswith("/"): + url = "https://api.github.com{}".format(url) + response = requests.get(url, headers=headers) + response.raise_for_status() + return response diff --git a/tests/test_get.py b/tests/test_get.py new file mode 100644 index 0000000..1c1dddd --- /dev/null +++ b/tests/test_get.py @@ -0,0 +1,93 @@ +from click.testing import CliRunner +from github_to_sqlite import cli +import pytest +import textwrap + + +@pytest.fixture +def mocked_paginated(requests_mock): + requests_mock.get( + "https://api.github.com/paginated", + json=[{"id": 1, "title": "Item 1"}, {"id": 2, "title": "Item 2"}], + headers={"link": '; rel="next"'}, + ) + requests_mock.get( + "https://api.github.com/paginated?page=2", + json=[{"id": 3, "title": "Item 3"}, {"id": 4, "title": "Item 4"}], + headers={"link": '; rel="prev"'}, + ) + + +@pytest.mark.parametrize("url", ["https://api.github.com/paginated", "/paginated"]) +def test_get(mocked_paginated, url): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(cli.cli, ["get", url]) + assert 0 == result.exit_code + expected = textwrap.dedent( + """ + [ + { + "id": 1, + "title": "Item 1" + }, + { + "id": 2, + "title": "Item 2" + } + ] + """ + ).strip() + assert result.output.strip() == expected + + +@pytest.mark.parametrize( + "nl,expected", + ( + ( + False, + textwrap.dedent( + """ + [ + { + "id": 1, + "title": "Item 1" + }, + { + "id": 2, + "title": "Item 2" + }, + { + "id": 3, + "title": "Item 3" + }, + { + "id": 4, + "title": "Item 4" + } + ]""" + ).strip(), + ), + ( + True, + textwrap.dedent( + """ + {"id": 1, "title": "Item 1"} + {"id": 2, "title": "Item 2"} + {"id": 3, "title": "Item 3"} + {"id": 4, "title": "Item 4"} + """ + ).strip(), + ), + ), +) +def test_get_paginate(mocked_paginated, nl, expected): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + cli.cli, + ["get", "https://api.github.com/paginated", "--paginate"] + + (["--nl"] if nl else []), + ) + assert 0 == result.exit_code + assert result.output.strip() == expected