From 5c4882d7fbffa480a8701d29b83af75021801e40 Mon Sep 17 00:00:00 2001 From: Ivan Ivanov Date: Fri, 28 Jul 2023 22:57:16 +0300 Subject: [PATCH] Make use of paginated API by default on `list-projects` and `list-jobs` --- qfieldcloud_sdk/cli.py | 71 +++++++------ qfieldcloud_sdk/interfaces.py | 18 +++- qfieldcloud_sdk/sdk.py | 183 +++++++++++++++++++--------------- qfieldcloud_sdk/utils.py | 9 -- tests/test_cli_client.py | 15 +-- 5 files changed, 156 insertions(+), 140 deletions(-) diff --git a/qfieldcloud_sdk/cli.py b/qfieldcloud_sdk/cli.py index f67d748..5ec6f90 100755 --- a/qfieldcloud_sdk/cli.py +++ b/qfieldcloud_sdk/cli.py @@ -1,7 +1,7 @@ import collections import platform from enum import Enum -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import click @@ -33,6 +33,26 @@ def list_commands(self, ctx): return self.commands +def paginated(command): + command = click.option( + "-o", + "--offset", + type=int, + default=None, + is_flag=False, + help="Offsets the given number of records in the paginated JSON response.", + )(command) + command = click.option( + "-l", + "--limit", + type=int, + default=None, + is_flag=False, + help="Limits the number of records to return in the paginated JSON response.", + )(command) + return command + + @click.group(cls=OrderedGroup) @click.option( "-U", @@ -149,22 +169,7 @@ def logout(ctx): @cli.command() -@click.option( - "-o", - "--offset", - type=int, - default=None, - is_flag=False, - help="Offsets the given number of projects in the paginated JSON response", -) -@click.option( - "-l", - "--limit", - type=int, - default=None, - is_flag=False, - help="Limits the number of projects to return in the paginated JSON response", -) +@paginated @click.option( "--include-public/--no-public", default=False, @@ -172,12 +177,15 @@ def logout(ctx): help="Includes the public project in the list. Default: False", ) @click.pass_context -def list_projects(ctx, **opts): +def list_projects(ctx, include_public, **opts): """List QFieldCloud projects.""" log("Listing projects…") - projects: List[Dict[str, Any]] = ctx.obj["client"].list_projects(**opts) + projects: List[Dict[str, Any]] = ctx.obj["client"].list_projects( + include_public, + sdk.Pagination(**opts), + ) if ctx.obj["format_json"]: print_json(projects) @@ -381,29 +389,18 @@ def delete_files(ctx, project_id, paths, throw_on_error): type=sdk.JobTypes, help="Job type. One of package, delta_apply or process_projectfile.", ) -@click.option( - "-o", - "--offset", - type=int, - default=None, - is_flag=False, - help="Offsets the given number of projects in the paginated JSON response", -) -@click.option( - "-l", - "--limit", - type=int, - default=None, - is_flag=False, - help="Limits the number of projects to return in the paginated JSON response", -) +@paginated @click.pass_context -def list_jobs(ctx, project_id, **opts): +def list_jobs(ctx, project_id, job_type: Optional[sdk.JobTypes], **opts): """List project jobs.""" log(f'Listing project "{project_id}" jobs…') - jobs: List[Dict[Any]] = ctx.obj["client"].list_jobs(project_id, **opts) + jobs: List[Dict[Any]] = ctx.obj["client"].list_jobs( + project_id, + job_type, + sdk.Pagination(**opts), + ) if ctx.obj["format_json"]: print_json(jobs) diff --git a/qfieldcloud_sdk/interfaces.py b/qfieldcloud_sdk/interfaces.py index da848c2..8a17988 100644 --- a/qfieldcloud_sdk/interfaces.py +++ b/qfieldcloud_sdk/interfaces.py @@ -15,14 +15,28 @@ def __getitem__(self, k: str) -> Any: class QfcMockResponse(requests.Response): def __init__(self, **kwargs): self.request_kwargs = kwargs + self.url = kwargs["url"] self.limit = kwargs.get("limit", 5) self.total = self.limit * 2 self.headers = { "X-Total-Count": self.total, - "X-Next-Page": "next_url", - "X-Previous-Page": "previous_url", } + limit = kwargs["params"].get("limit") + offset = kwargs["params"].get("offset", 0) + prev_url = None + next_url = None + if limit: + if offset == 0: + prev_url = None + next_url = f"{self.url}?limit={limit}&offset={limit}" + else: + prev_url = f"{self.url}?limit={limit}&offset=0" + next_url = None + + self.headers["X-Previous-Page"] = prev_url + self.headers["X-Next-Page"] = next_url + def json(self) -> Union[QfcMockItem, List[QfcMockItem]]: if self.request_kwargs["method"] == "GET": return [QfcMockItem(id=n) for n in range(self.total)] diff --git a/qfieldcloud_sdk/sdk.py b/qfieldcloud_sdk/sdk.py index 79fd35c..01c865d 100644 --- a/qfieldcloud_sdk/sdk.py +++ b/qfieldcloud_sdk/sdk.py @@ -4,14 +4,15 @@ import sys from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union +from urllib import parse as urlparse import requests import urllib3 from requests.adapters import HTTPAdapter, Retry from .interfaces import QfcException, QfcRequest, QfcRequestException -from .utils import get_numeric_params, log +from .utils import log logger = logging.getLogger(__file__) @@ -27,6 +28,9 @@ __version__ = "dev" +DEFAULT_PAGINATION_LIMIT = 20 + + class FileTransferStatus(str, Enum): PENDING = "PENDING" SUCCESS = "SUCCESS" @@ -44,6 +48,21 @@ class JobTypes(str, Enum): PROCESS_PROJECTFILE = "process_projectfile" +class Pagination: + limit = None + offset = None + + def __init__( + self, limit: Optional[int] = None, offset: Optional[int] = None + ) -> None: + self.limit = limit + self.offset = offset + + @property + def is_empty(self): + return self.limit is None and self.offset is None + + class Client: def __init__( self, url: str = None, verify_ssl: bool = None, token: str = None @@ -109,35 +128,21 @@ def logout(self) -> None: def list_projects( self, include_public: Optional[bool] = False, - limit: Optional[int] = None, - offset: Optional[int] = None, + pagination: Pagination = Pagination(), **kwargs, ) -> List[Dict[str, Any]]: """ - Returns a paginated lists of projects accessible to the current user, + Returns a list of projects accessible to the current user, their own and optionally the public ones. """ - log( - """ - API CHANGE NOTICE: You have called an API endpoint whose results will be paginated in a near release. - You will be able to use `--offset` and `--limit` to take advantage of it. - """ - ) - params = { "include-public": int(include_public), } - if offset: - params["offset"] = offset - - if include_public: - params["limit"] = min(50, int(limit) or 0) - elif limit: - params["limit"] = limit - - resp = self._request("GET", "projects", params=params) - return self._serialize_paginated_results(resp) + payload = self._request_json( + "GET", "projects", params=params, pagination=pagination + ) + return payload def list_remote_files( self, project_id: str, skip_metadata: bool = True @@ -322,40 +327,22 @@ def download_project( def list_jobs( self, project_id: str, - limit: Optional[int] = None, - offset: Optional[int] = None, job_type: JobTypes = None, + pagination: Pagination = Pagination(), ) -> List[Dict[str, Any]]: """ Returns a paginated lists of jobs accessible to the user. """ - log( - """ - API CHANGE NOTICE: You have called an API endpoint whose results will be paginated in a near release. - You will be able to use `--offset` and `--limit` to take advantage of it. - """ - ) - - params = {} - - if limit: - params["limit"] = limit - - if offset: - params["offset"] = offset - - resp = self._request( + return self._request_json( "GET", "jobs/", { "project_id": project_id, "type": job_type.value if job_type else None, }, - params=params, + pagination=pagination, ) - return self._serialize_paginated_results(resp) - def job_trigger( self, project_id: str, job_type: JobTypes, force: bool = False ) -> Dict[str, Any]: @@ -688,6 +675,68 @@ def list_local_files( return files + def _request_json( + self, + method: str, + path: str, + data: Any = None, + params: Dict[str, str] = {}, + headers: Dict[str, str] = {}, + files: Dict[str, Any] = None, + stream: bool = False, + skip_token: bool = False, + allow_redirects=None, + pagination: Pagination = Pagination(), + ) -> Union[List, Dict]: + result = None + is_empty_pagination = pagination.is_empty + + while True: + resp = self._request( + method, + path, + data, + params, + headers, + files, + stream, + skip_token, + allow_redirects, + pagination, + ) + + payload = resp.json() + + if isinstance(payload, list): + if result: + result += payload + else: + result = payload + elif isinstance(payload, dict): + if result: + result = {**result, **payload} + else: + result = payload + else: + raise NotImplementedError( + "Unsupported data type for paginated response." + ) + + if not is_empty_pagination: + break + + next_url = resp.headers.get("X-Next-Page") + if not next_url: + break + + query_params = urlparse.parse_qs(urlparse.urlparse(next_url).query) + pagination = Pagination( + limit=query_params["limit"], + offset=query_params["offset"], + ) + + return result + def _request( self, method: str, @@ -699,6 +748,7 @@ def _request( stream: bool = False, skip_token: bool = False, allow_redirects=None, + pagination: Optional[Pagination] = None, ) -> requests.Response: method = method.upper() headers_copy = {**headers} @@ -723,6 +773,15 @@ def _request( path = self.url + path + if pagination: + limit = pagination.limit or DEFAULT_PAGINATION_LIMIT + offset = pagination.offset or 0 + params = { + **params, + "limit": limit, + "offset": offset, + } + request_params = { "method": method, "url": path, @@ -752,41 +811,3 @@ def _request( raise QfcRequestException(response) from err return response - - @staticmethod - def _serialize_paginated_results( - response: requests.Response, - ) -> List[Dict[str, Any]]: - """Serialize results. Notify en passant users if results are paginated.""" - total_count_header = response.headers.get("X-Total-Count") - - if not total_count_header: - # We know that no server-side pagination has occurred. Nothing to notify the user about. Serialize and return. - return response.json() - - total_count = int(total_count_header) - previous = response.headers.get("X-Previous-Page") - next = response.headers.get("X-Next-Page") - results = response.json() - results_count = len(results) - - if results_count < total_count: - # We know that server-side pagination occurred and there are more items to get. - # So let the user know about that. - log(f"{len(results)} out of {total_count}. Results are paginated.") - - if previous: - previous_offset, limit = get_numeric_params( - previous, ("offset", "limit") - ) - log( - f"Use `--offset {previous_offset} and --limit {limit}` to see the previous page" - ) - - if next: - next_offset, limit = get_numeric_params(next, ("offset", "limit")) - print( - f"Use `--offset {next_offset} and --limit {limit}` to see the next page" - ) - - return results diff --git a/qfieldcloud_sdk/utils.py b/qfieldcloud_sdk/utils.py index 56ffad6..4b905a9 100644 --- a/qfieldcloud_sdk/utils.py +++ b/qfieldcloud_sdk/utils.py @@ -1,8 +1,6 @@ import hashlib import json import sys -from typing import Iterable, Tuple -from urllib.parse import parse_qs, urlparse def print_json(data): @@ -25,10 +23,3 @@ def get_md5sum(filename: str) -> str: hasher.update(buf) buf = f.read(BLOCKSIZE) return hasher.hexdigest() - - -def get_numeric_params(url: str, params: Iterable[str]) -> Tuple[int]: - """Extract numeric parameters from url GET query""" - parsed_url = urlparse(url) - parsed_query = parse_qs(parsed_url.query) - return tuple(int(parsed_query[k][0]) for k in params) diff --git a/tests/test_cli_client.py b/tests/test_cli_client.py index 78d3062..c2201bb 100644 --- a/tests/test_cli_client.py +++ b/tests/test_cli_client.py @@ -3,8 +3,7 @@ from click.testing import CliRunner from qfieldcloud_sdk.cli import QFIELDCLOUD_DEFAULT_URL, cli -from qfieldcloud_sdk.sdk import Client -from qfieldcloud_sdk.utils import get_numeric_params, log +from qfieldcloud_sdk.sdk import Client, Pagination class TestSDK(unittest.TestCase): @@ -12,18 +11,14 @@ class TestSDK(unittest.TestCase): def setUpClass(cls): cls.client = Client(QFIELDCLOUD_DEFAULT_URL) - def test_parse_params(self): - url = "https//my_service.org/api/?limit=10&offset=5" - limit, offset = get_numeric_params(url, ("limit", "offset")) - self.assertEqual(limit, 10) - self.assertEqual(offset, 5) - def test_paginated_list_projects(self): results = self.client.list_projects(limit=20) self.assertTrue(0 < len(results) and len(results) <= 20) def test_paginated_list_projects_include_public(self): - results = self.client.list_projects(include_public=True, limit=200) + results = self.client.list_projects( + include_public=True, pagination=Pagination(limit=200) + ) self.assertTrue(0 < len(results) and len(results) <= 50) @@ -46,7 +41,6 @@ def test_list_projects(self): catch_exceptions=False, ) self.assertEqual(result.exit_code, 0) - log(result.output) def test_list_jobs(self): result = self.runner.invoke( @@ -55,4 +49,3 @@ def test_list_jobs(self): catch_exceptions=False, ) self.assertEqual(result.exit_code, 0) - log(result.output)