From 20118b9e18fd3a23ba0e6e9bf47645721f8b9ea3 Mon Sep 17 00:00:00 2001 From: Joohwan Oh Date: Sun, 4 Aug 2019 14:09:34 -0700 Subject: [PATCH] Revamp the library for 5.0.0. --- .travis.yml | 22 +- README.rst | 30 +- arango/api.py | 1 - arango/aql.py | 83 ++++-- arango/client.py | 131 ++++----- arango/collection.py | 562 ++++++++++++++------------------------ arango/connection.py | 121 ++++++-- arango/cursor.py | 18 +- arango/database.py | 423 +++++++++++++++++++++------- arango/exceptions.py | 48 +++- arango/executor.py | 287 +++++++++---------- arango/foxx.py | 8 +- arango/http.py | 44 ++- arango/job.py | 55 +--- arango/request.py | 55 ++-- arango/resolver.py | 49 ++++ arango/response.py | 27 +- arango/version.py | 2 +- docs/admin.rst | 3 - docs/analyzer.rst | 35 +++ docs/aql.rst | 6 +- docs/async.rst | 14 +- docs/batch.rst | 6 +- docs/cursor.rst | 39 --- docs/database.rst | 2 +- docs/document.rst | 2 +- docs/errors.rst | 3 - docs/foxx.rst | 7 +- docs/http.rst | 102 ++----- docs/index.rst | 11 +- docs/indexes.rst | 8 +- docs/logging.rst | 12 +- docs/overview.rst | 2 +- docs/pregel.rst | 4 +- docs/serializer.rst | 22 ++ docs/specs.rst | 8 - docs/threading.rst | 2 - docs/transaction.rst | 322 ++++------------------ docs/view.rst | 44 ++- docs/wal.rst | 6 +- tests/conftest.py | 64 +++-- tests/executors.py | 37 ++- tests/helpers.py | 39 ++- tests/test_analyzer.py | 60 ++++ tests/test_aql.py | 60 ++-- tests/test_async.py | 2 +- tests/test_batch.py | 4 + tests/test_client.py | 75 +++-- tests/test_collection.py | 95 ++++--- tests/test_cursor.py | 12 +- tests/test_database.py | 17 +- tests/test_document.py | 351 +++++++----------------- tests/test_exception.py | 9 +- tests/test_foxx.py | 5 +- tests/test_graph.py | 53 ++-- tests/test_index.py | 154 +++++++---- tests/test_permission.py | 19 +- tests/test_pregel.py | 17 +- tests/test_request.py | 46 +--- tests/test_resolver.py | 34 +++ tests/test_response.py | 30 +- tests/test_task.py | 1 - tests/test_transaction.py | 304 +++++++-------------- tests/test_user.py | 11 +- tests/test_view.py | 121 ++++++-- 65 files changed, 2121 insertions(+), 2125 deletions(-) create mode 100644 arango/resolver.py create mode 100644 docs/analyzer.rst create mode 100644 docs/serializer.rst create mode 100644 tests/test_analyzer.py create mode 100644 tests/test_resolver.py diff --git a/.travis.yml b/.travis.yml index 9cfd1833..62e434d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,25 @@ sudo: false language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 +matrix: + include: + - python: 2.7 + - python: 3.4 + - python: 3.5 + - python: 3.6 + - python: 3.7 + dist: xenial + sudo: true services: - docker before_install: - - docker run --name arango -d -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd arangodb/arangodb:3.4.0 + - docker run --name arango -d -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd arangodb/arangodb:3.5.0 - docker cp tests/static/service.zip arango:/tmp/service.zip install: - - pip install flake8 mock pytest pytest-cov python-coveralls sphinx sphinx_rtd_theme + - pip install flake8 mock + - pip install pytest==3.5.1 + - pip install pytest-cov==2.5.1 + - pip install python-coveralls==2.9.1 + - pip install sphinx sphinx_rtd_theme - pip install . script: - python -m flake8 diff --git a/README.rst b/README.rst index c8385552..ef706c56 100644 --- a/README.rst +++ b/README.rst @@ -22,11 +22,11 @@ :target: https://coveralls.io/github/joowani/python-arango?branch=master :alt: Test Coverage -.. image:: https://img.shields.io/github/issues/joowani/python-arango.svg +.. image:: https://img.shields.io/github/issues/joowani/python-arango.svg :target: https://github.com/joowani/python-arango/issues :alt: Issues Open -.. image:: https://img.shields.io/badge/license-MIT-blue.svg +.. image:: https://img.shields.io/badge/license-MIT-blue.svg :target: https://raw.githubusercontent.com/joowani/python-arango/master/LICENSE :alt: MIT License @@ -37,23 +37,25 @@ Welcome to the GitHub page for **python-arango**, a Python driver for ArangoDB_. Announcements ============= -- Python-arango version `4.0.0`_ is now out! -- Please see the releases_ page for latest updates. +- Python-arango version `5.0.0`_ is finally up! This release supports ArangoDB + version 3.5+ only. It also breaks backward-compatibility and you must make + changes in your application code. Please see the releases_ page for details. Features ======== -- Clean Pythonic interface. -- Lightweight. -- High ArangoDB REST API coverage. +- Pythonic interface +- Lightweight +- High API coverage Compatibility ============= -- Python versions 2.7, 3.4, 3.5 and 3.6 are supported. -- Python-arango 4.x supports ArangoDB 3.3+ (recommended). -- Python-arango 3.x supports ArangoDB 3.0 ~ 3.2 only. -- Python-arango 2.x supports ArangoDB 1.x ~ 2.x only. +- Python versions 2.7, 3.4, 3.5, 3.6 and 3.7 are supported +- Python-arango 5.x supports ArangoDB 3.5+ +- Python-arango 4.x supports ArangoDB 3.3 ~ 3.4 only +- Python-arango 3.x supports ArangoDB 3.0 ~ 3.2 only +- Python-arango 2.x supports ArangoDB 1.x ~ 2.x only Installation ============ @@ -83,7 +85,7 @@ Here is a simple usage example: from arango import ArangoClient # Initialize the client for ArangoDB. - client = ArangoClient(protocol='http', host='localhost', port=8529) + client = ArangoClient(hosts='http://localhost:8529') # Connect to "_system" database as root user. sys_db = client.db('_system', username='root', password='passwd') @@ -117,7 +119,7 @@ Here is another example with graphs: from arango import ArangoClient # Initialize the client for ArangoDB. - client = ArangoClient(protocol='http', host='localhost', port=8529) + client = ArangoClient(hosts='http://localhost:8529') # Connect to "test" database as root user. db = client.db('test', username='root', password='passwd') @@ -169,7 +171,7 @@ Contributing Please take a look at this page_ before submitting a pull request. Thanks! .. _ArangoDB: https://www.arangodb.com -.. _4.0.0: https://github.com/joowani/python-arango/releases/tag/4.0.0 +.. _5.0.0: https://github.com/joowani/python-arango/releases/tag/5.0.0 .. _releases: https://github.com/joowani/python-arango/releases .. _PyPi: https://pypi.python.org/pypi/python-arango .. _GitHub: https://github.com/joowani/python-arango diff --git a/arango/api.py b/arango/api.py index bf4a70c0..bb86e187 100644 --- a/arango/api.py +++ b/arango/api.py @@ -15,7 +15,6 @@ class APIWrapper(object): def __init__(self, connection, executor): self._conn = connection self._executor = executor - self._is_transaction = self.context == 'transaction' @property def db_name(self): diff --git a/arango/aql.py b/arango/aql.py index 77e2f29e..552da506 100644 --- a/arango/aql.py +++ b/arango/aql.py @@ -1,7 +1,5 @@ from __future__ import absolute_import, unicode_literals -from json import dumps - __all__ = ['AQL', 'AQLQueryCache'] from arango.api import APIWrapper @@ -42,7 +40,7 @@ def __repr__(self): return ''.format(self._conn.db_name) # noinspection PyMethodMayBeStatic - def _format_tracking(self, body): + def _format_tracking_properties(self, body): """Format the tracking properties. :param body: Response body. @@ -237,10 +235,10 @@ def execute(self, enterprise version of ArangoDB. :type satellite_sync_wait: int | float :param read_collections: Names of collections read during query - execution. Required for :doc:`transactions `. + execution. This parameter is deprecated. :type read_collections: [str | unicode] :param write_collections: Names of collections written to during query - execution. Required for :doc:`transactions `. + execution. This parameter is deprecated. :type write_collections: [str | unicode] :param stream: If set to True, query is executed in streaming fashion: query result is not stored server-side but calculated on the fly. @@ -310,17 +308,10 @@ def execute(self, data['options'] = options data.update(options) - command = 'db._query({}, {}, {}).toArray()'.format( - dumps(query), - dumps(bind_vars), - dumps(data), - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/cursor', data=data, - command=command, read=read_collections, write=write_collections ) @@ -425,7 +416,7 @@ def tracking(self): def response_handler(resp): if not resp.is_success: raise AQLQueryTrackingGetError(resp, request) - return self._format_tracking(resp.body) + return self._format_tracking_properties(resp.body) return self._execute(request, response_handler) @@ -465,7 +456,7 @@ def set_tracking(self, def response_handler(resp): if not resp.is_success: raise AQLQueryTrackingSetError(resp, request) - return self._format_tracking(resp.body) + return self._format_tracking_properties(resp.body) return self._execute(request, response_handler) @@ -563,6 +554,28 @@ class AQLQueryCache(APIWrapper): def __repr__(self): return ''.format(self._conn.db_name) + # noinspection PyMethodMayBeStatic + def _format_cache_properties(self, body): + """Format the query cache properties. + + :param body: Response body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + body.pop('code', None) + body.pop('error', None) + + if 'maxResults' in body: + body['max_results'] = body.pop('maxResults') + if 'maxResultsSize' in body: + body['max_results_size'] = body.pop('maxResultsSize') + if 'maxEntrySize' in body: + body['max_entry_size'] = body.pop('maxEntrySize') + if 'includeSystem' in body: + body['include_system'] = body.pop('includeSystem') + return body + def properties(self): """Return the query cache properties. @@ -578,21 +591,32 @@ def properties(self): def response_handler(resp): if not resp.is_success: raise AQLCachePropertiesError(resp, request) - return { - 'mode': resp.body['mode'], - 'limit': resp.body['maxResults'] - } + return self._format_cache_properties(resp.body) return self._execute(request, response_handler) - def configure(self, mode=None, limit=None): + def configure(self, + mode=None, + max_results=None, + max_results_size=None, + max_entry_size=None, + include_system=None): """Configure the query cache properties. :param mode: Operation mode. Allowed values are "off", "on" and "demand". :type mode: str | unicode - :param limit: Max number of query results to be stored. - :type limit: int + :param max_results: Max number of query results stored per + database-specific cache. + :type max_results: int + :param max_results_size: Max cumulative size of query results stored + per database-specific cache. + :type max_results_size: int + :param max_entry_size: Max entry size of each query result stored per + database-specific cache. + :type max_entry_size: int + :param include_system: Store results of queries in system collections. + :type include_system: bool :return: Query cache properties. :rtype: dict :raise arango.exceptions.AQLCacheConfigureError: If operation fails. @@ -600,8 +624,14 @@ def configure(self, mode=None, limit=None): data = {} if mode is not None: data['mode'] = mode - if limit is not None: - data['maxResults'] = limit + if max_results is not None: + data['maxResults'] = max_results + if max_results_size is not None: + data['maxResultsSize'] = max_results_size + if max_entry_size is not None: + data['maxEntrySize'] = max_entry_size + if include_system is not None: + data['includeSystem'] = include_system request = Request( method='put', @@ -612,10 +642,7 @@ def configure(self, mode=None, limit=None): def response_handler(resp): if not resp.is_success: raise AQLCacheConfigureError(resp, request) - return { - 'mode': resp.body['mode'], - 'limit': resp.body['maxResults'] - } + return self._format_cache_properties(resp.body) return self._execute(request, response_handler) @@ -642,7 +669,7 @@ def clear(self): """Clear the query cache. :return: True if query cache was cleared successfully. - :rtype: dict + :rtype: bool :raise arango.exceptions.AQLCacheClearError: If operation fails. """ request = Request( diff --git a/arango/client.py b/arango/client.py index c1e17668..fd0751ca 100644 --- a/arango/client.py +++ b/arango/client.py @@ -1,87 +1,92 @@ from __future__ import absolute_import, unicode_literals +import json + +from six import string_types + __all__ = ['ArangoClient'] from arango.connection import Connection from arango.database import StandardDatabase from arango.exceptions import ServerConnectionError +from arango.http import DefaultHTTPClient +from arango.resolver import ( + SingleHostResolver, + RandomHostResolver, + RoundRobinHostResolver +) from arango.version import __version__ class ArangoClient(object): """ArangoDB client. - :param protocol: Internet transfer protocol (default: "http"). - :type protocol: str | unicode - :param host: ArangoDB host (default: "127.0.0.1"). - :type host: str | unicode - :param port: ArangoDB port (default: 8529). - :type port: int + :param hosts: Host URL or list of URLs (coordinators in a cluster). + :type hosts: [str | unicode] + :param host_resolver: Host resolver. This parameter used for clusters (when + multiple host URLs are provided). Accepted values are "roundrobin" and + "random". Any other value defaults to round robin. + :type host_resolver: str | unicode :param http_client: User-defined HTTP client. :type http_client: arango.http.HTTPClient + :param serializer: User-defined JSON serializer. Must be a callable + which takes a JSON data type object as its only argument and return + the serialized string. If not given, ``json.dumps`` is used by default. + :type serializer: callable + :param deserializer: User-defined JSON de-serializer. Must be a callable + which takes a JSON serialized string as its only argument and return + the de-serialized object. If not given, ``json.loads`` is used by + default. + :type deserializer: callable """ def __init__(self, - protocol='http', - host='127.0.0.1', - port=8529, - http_client=None): - self._protocol = protocol.strip('/') - self._host = host.strip('/') - self._port = int(port) - self._url = '{}://{}:{}'.format(protocol, host, port) - self._http_client = http_client + hosts='http://127.0.0.1:8529', + host_resolver='roundrobin', + http_client=None, + serializer=json.dumps, + deserializer=json.loads): + if isinstance(hosts, string_types): + self._hosts = [host.strip('/') for host in hosts.split(',')] + else: + self._hosts = [host.strip('/') for host in hosts] + + host_count = len(self._hosts) + if host_count == 1: + self._host_resolver = SingleHostResolver() + elif host_resolver == 'random': + self._host_resolver = RandomHostResolver(host_count) + else: + self._host_resolver = RoundRobinHostResolver(host_count) + + self._http = http_client or DefaultHTTPClient() + self._serializer = serializer + self._deserializer = deserializer + self._sessions = [self._http.create_session(h) for h in self._hosts] def __repr__(self): - return ''.format(self._url) - - @property - def version(self): - """Return the client version. - - :return: Client version. - :rtype: str | unicode - """ - return __version__ - - @property - def protocol(self): - """Return the internet transfer protocol (e.g. "http"). - - :return: Internet transfer protocol. - :rtype: str | unicode - """ - return self._protocol + return ''.format(','.join(self._hosts)) @property - def host(self): - """Return the ArangoDB host. + def hosts(self): + """Return the list of ArangoDB host URLs. - :return: ArangoDB host. - :rtype: str | unicode + :return: List of ArangoDB host URLs. + :rtype: [str | unicode] """ - return self._host + return self._hosts @property - def port(self): - """Return the ArangoDB port. - - :return: ArangoDB port. - :rtype: int - """ - return self._port - - @property - def base_url(self): - """Return the ArangoDB base URL. + def version(self): + """Return the client version. - :return: ArangoDB base URL. + :return: Client version. :rtype: str | unicode """ - return self._url + return __version__ def db(self, name='_system', username='root', password='', verify=False): - """Connect to a database and return the database API wrapper. + """Connect to an ArangoDB database and return the database API wrapper. :param name: Database name. :type name: str | unicode @@ -94,23 +99,25 @@ def db(self, name='_system', username='root', password='', verify=False): :return: Standard database API wrapper. :rtype: arango.database.StandardDatabase :raise arango.exceptions.ServerConnectionError: If **verify** was set - to True and the connection to ArangoDB fails. + to True and the connection fails. """ connection = Connection( - url=self._url, - db=name, + hosts=self._hosts, + host_resolver=self._host_resolver, + sessions=self._sessions, + db_name=name, username=username, password=password, - http_client=self._http_client + http_client=self._http, + serializer=self._serializer, + deserializer=self._deserializer ) - database = StandardDatabase(connection) - - if verify: # Check the server connection by making a read API call + if verify: try: - database.ping() + connection.ping() except ServerConnectionError as err: raise err except Exception as err: raise ServerConnectionError('bad connection: {}'.format(err)) - return database + return StandardDatabase(connection) diff --git a/arango/collection.py b/arango/collection.py index a46e462b..fdaf927b 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -4,8 +4,6 @@ from numbers import Number -from json import dumps - from arango.api import APIWrapper from arango.cursor import Cursor from arango.exceptions import ( @@ -13,6 +11,8 @@ CollectionConfigureError, CollectionLoadError, CollectionPropertiesError, + CollectionRecalculateCountError, + CollectionResponsibleShardError, CollectionRenameError, CollectionRevisionError, CollectionRotateJournalError, @@ -95,7 +95,7 @@ def _get_status_text(self, code): # pragma: no cover return None if code is None else self.statuses[code] def _format_properties(self, body): # pragma: no cover - """Format the collection properties. + """Format collection properties. :param body: Response body. :type body: dict @@ -217,10 +217,6 @@ def _prep_from_doc(self, document, rev, check_rev): if not check_rev or rev is None: return doc_id, doc_id, {} - elif self._is_transaction: - body = document.copy() - body['_rev'] = rev - return doc_id, body, {'If-Match': rev} else: return doc_id, doc_id, {'If-Match': rev} else: @@ -231,9 +227,6 @@ def _prep_from_doc(self, document, rev, check_rev): if not check_rev or rev is None: return doc_id, doc_id, {} - elif self._is_transaction: - body = {'_id': doc_id, '_rev': rev} - return doc_id, body, {'If-Match': rev} else: return doc_id, doc_id, {'If-Match': rev} @@ -278,6 +271,48 @@ def name(self): """ return self._name + def recalculate_count(self): + """Recalculate the document count. + + :return: True if recalculation was successful. + :rtype: bool + :raise arango.exceptions.CollectionRecalculateCountError: If operation + fails. + """ + request = Request( + method='put', + endpoint='/_api/collection/{}/recalculateCount'.format(self.name) + ) + + def response_handler(resp): + if not resp.is_success: + raise CollectionRecalculateCountError(resp, request) + return True + + return self._execute(request, response_handler) + + def responsible_shard(self, document): # pragma: no cover + """Return the ID of the shard responsible for given **document**. + + If the document does not exist, return the shard that would be + responsible. + + :return: Shard ID + """ + request = Request( + method='put', + endpoint='/_api/collection/{}/responsibleShard'.format(self.name), + data=document, + read=self.name + ) + + def response_handler(resp): + if resp.is_success: + return resp.body['shardId'] + raise CollectionResponsibleShardError(resp, request) + + return self._execute(request, response_handler) + def rename(self, new_name): """Rename the collection. @@ -316,7 +351,6 @@ def properties(self): request = Request( method='get', endpoint='/_api/collection/{}/properties'.format(self.name), - command='db.{}.properties()'.format(self.name), read=self.name ) @@ -367,7 +401,6 @@ def statistics(self): request = Request( method='get', endpoint='/_api/collection/{}/figures'.format(self.name), - command='db.{}.figures()'.format(self.name), read=self.name ) @@ -396,6 +429,12 @@ def response_handler(resp): stats['waiting_for'] = stats.pop('waitingFor') if 'documentsSize' in stats: # pragma: no cover stats['documents_size'] = stats.pop('documentsSize') + if 'cacheInUse' in stats: # pragma: no cover + stats['cache_in_use'] = stats.pop('cacheInUse') + if 'cacheSize' in stats: # pragma: no cover + stats['cache_size'] = stats.pop('cacheSize') + if 'cacheUsage' in stats: # pragma: no cover + stats['cache_usage'] = stats.pop('cacheUsage') if 'uncollectedLogfileEntries' in stats: # pragma: no cover stats['uncollected_logfile_entries'] = \ stats.pop('uncollectedLogfileEntries') @@ -413,15 +452,12 @@ def revision(self): request = Request( method='get', endpoint='/_api/collection/{}/revision'.format(self.name), - command='db.{}.revision()'.format(self.name), read=self.name ) def response_handler(resp): if not resp.is_success: raise CollectionRevisionError(resp, request) - if self._is_transaction: - return str(resp.body) return resp.body['revision'] return self._execute(request, response_handler) @@ -440,9 +476,7 @@ def checksum(self, with_rev=False, with_data=False): request = Request( method='get', endpoint='/_api/collection/{}/checksum'.format(self.name), - params={'withRevision': with_rev, 'withData': with_data}, - command='db.{}.checksum()'.format(self.name), - read=self.name + params={'withRevision': with_rev, 'withData': with_data} ) def response_handler(resp): @@ -513,14 +547,12 @@ def truncate(self): """Delete all documents in the collection. :return: True if collection was truncated successfully. - :rtype: dict + :rtype: bool :raise arango.exceptions.CollectionTruncateError: If operation fails. """ request = Request( method='put', - endpoint='/_api/collection/{}/truncate'.format(self.name), - command='db.{}.truncate()'.format(self.name), - write=self.name + endpoint='/_api/collection/{}/truncate'.format(self.name) ) def response_handler(resp): @@ -539,16 +571,12 @@ def count(self): """ request = Request( method='get', - endpoint='/_api/collection/{}/count'.format(self.name), - command='db.{}.count()'.format(self.name), - read=self.name + endpoint='/_api/collection/{}/count'.format(self.name) ) def response_handler(resp): if not resp.is_success: raise DocumentCountError(resp, request) - if self._is_transaction: - return resp.body return resp.body['count'] return self._execute(request, response_handler) @@ -572,16 +600,10 @@ def has(self, document, rev=None, check_rev=True): """ handle, body, headers = self._prep_from_doc(document, rev, check_rev) - command = 'db.{}.exists({})'.format( - self.name, - dumps(body) - ) if self._is_transaction else None - request = Request( method='get', endpoint='/_api/document/{}'.format(handle), headers=headers, - command=command, read=self.name ) @@ -607,7 +629,6 @@ def ids(self): method='put', endpoint='/_api/simple/all-keys', data={'collection': self.name, 'type': 'id'}, - command='db.{}.toArray().map(d => d._id)'.format(self.name), read=self.name ) @@ -629,7 +650,6 @@ def keys(self): method='put', endpoint='/_api/simple/all-keys', data={'collection': self.name, 'type': 'key'}, - command='db.{}.toArray().map(d => d._key)'.format(self.name), read=self.name ) @@ -660,24 +680,14 @@ def all(self, skip=None, limit=None): if limit is not None: data['limit'] = limit - command = 'db.{}.all(){}{}.toArray()'.format( - self.name, - '' if skip is None else '.skip({})'.format(skip), - '' if limit is None else '.limit({})'.format(limit), - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/all', data=data, - command=command, read=self.name ) def response_handler(resp): - # TODO workaround for a bug in ArangoDB - if self._is_transaction and limit == 0: - return Cursor(self._conn, []) if not resp.is_success: raise DocumentGetError(resp, request) return Cursor(self._conn, resp.body) @@ -692,7 +702,7 @@ def export(self, flush_wait=None, ttl=None, filter_fields=None, - filter_type='include'): # pragma: no cover + filter_type='include'): """Export all documents in the collection using a server cursor. :param flush: If set to True, flush the write-ahead log prior to the @@ -771,18 +781,10 @@ def find(self, filters, skip=None, limit=None): if limit is not None: data['limit'] = limit - command = 'db.{}.byExample({}){}{}.toArray()'.format( - self.name, - dumps(filters), - '' if skip is None else '.skip({})'.format(skip), - '' if limit is None else '.limit({})'.format(limit), - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/by-example', data=data, - command=command, read=self.name ) @@ -793,7 +795,7 @@ def response_handler(resp): return self._execute(request, response_handler) - def find_near(self, latitude, longitude, limit=None): # pragma: no cover + def find_near(self, latitude, longitude, limit=None): """Return documents near a given coordinate. Documents returned are sorted according to distance, with the nearest @@ -828,16 +830,6 @@ def find_near(self, latitude, longitude, limit=None): # pragma: no cover if limit is not None: bind_vars['limit'] = limit - if not self._is_transaction: - command = 'db.{}.near({},{}){}.toArray()'.format( - self.name, - latitude, - longitude, - '' if limit is None else '.limit({})'.format(limit), - ) - else: - command = None - request = Request( method='post', endpoint='/_api/cursor', @@ -846,7 +838,6 @@ def find_near(self, latitude, longitude, limit=None): # pragma: no cover 'bindVars': bind_vars, 'count': True }, - command=command, read=self.name ) @@ -900,15 +891,6 @@ def find_in_range(self, RETURN doc """ - command = 'db.{}.range({},{},{}){}{}.toArray()'.format( - self.name, - dumps(field), - dumps(lower), - dumps(upper), - '' if skip is None else '.skip({})'.format(skip), - '' if limit is None else '.limit({})'.format(limit), - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/cursor', @@ -917,14 +899,10 @@ def find_in_range(self, 'bindVars': bind_vars, 'count': True }, - command=command, read=self.name ) def response_handler(resp): - # TODO workaround for a bug in ArangoDB - if self._is_transaction and limit == 0: - return Cursor(self._conn, []) if not resp.is_success: raise DocumentGetError(resp, request) return Cursor(self._conn, resp.body) @@ -968,13 +946,6 @@ def find_in_radius(self, latitude, longitude, radius, distance_field=None): if distance_field is not None: bind_vars['distance'] = distance_field - command = 'db.{}.within({},{},{}).toArray()'.format( - self.name, - latitude, - longitude, - radius - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/cursor', @@ -983,7 +954,6 @@ def find_in_radius(self, latitude, longitude, radius, distance_field=None): 'bindVars': bind_vars, 'count': True }, - command=command, read=self.name ) @@ -1044,21 +1014,10 @@ def find_in_box(self, if index is not None: data['geo'] = self._name + '/' + index - command = 'db.{}.withinRectangle({},{},{},{}){}{}.toArray()'.format( - self.name, - latitude1, - longitude1, - latitude2, - longitude2, - '' if skip is None else '.skip({})'.format(skip), - '' if limit is None else '.limit({})'.format(limit), - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/within-rectangle', data=data, - command=command, read=self.name ) @@ -1097,25 +1056,14 @@ def find_by_text(self, field, query, limit=None): RETURN doc """.format('' if limit is None else ', @limit') - command = 'db.{}.fulltext({},{}){}.toArray()'.format( - self.name, - dumps(field), - dumps(query), - '' if limit is None else '.limit({})'.format(limit), - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/cursor', data={'query': aql, 'bindVars': bind_vars, 'count': True}, - command=command, read=self.name ) def response_handler(resp): - # TODO workaround for a bug in ArangoDB - if self._is_transaction and limit == 0: - return Cursor(self._conn, []) if not resp.is_success: raise DocumentGetError(resp, request) return Cursor(self._conn, resp.body) @@ -1137,26 +1085,17 @@ def get_many(self, documents): for doc in documents ] - command = 'db.{}.document({})'.format( - self.name, - dumps(handles) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/lookup-by-keys', data={'collection': self.name, 'keys': handles}, - command=command, read=self.name ) def response_handler(resp): if not resp.is_success: raise DocumentGetError(resp, request) - if self._is_transaction: - docs = resp.body - else: - docs = resp.body['documents'] + docs = resp.body['documents'] return [doc for doc in docs if '_id' in doc] return self._execute(request, response_handler) @@ -1172,15 +1111,12 @@ def random(self): method='put', endpoint='/_api/simple/any', data={'collection': self.name}, - command='db.{}.any()'.format(self.name), read=self.name ) def response_handler(resp): if not resp.is_success: raise DocumentGetError(resp, request) - if self._is_transaction: - return resp.body return resp.body['document'] return self._execute(request, response_handler) @@ -1189,6 +1125,37 @@ def response_handler(resp): # Index Management # #################### + # noinspection PyMethodMayBeStatic + def _format_index_details(self, body): # pragma: no cover + """Format index details. + + :param body: Response body. + :type body: dict + :return: Formatted body. + :rtype: dict + """ + body.pop('error', None) + body.pop('code', None) + + if 'id' in body: + body['id'] = body['id'].split('/', 1)[1] + if 'minLength' in body: + body['min_length'] = body.pop('minLength') + if 'geoJson' in body: + body['geo_json'] = body.pop('geoJson') + if 'ignoreNull' in body: + body['ignore_none'] = body.pop('ignoreNull') + if 'selectivityEstimate' in body: + body['selectivity'] = body.pop('selectivityEstimate') + if 'isNewlyCreated' in body: + body['new'] = body.pop('isNewlyCreated') + if 'expireAfter' in body: + body['expiry_time'] = body.pop('expireAfter') + if 'inBackground' in body: + body['in_background'] = body.pop('inBackground') + + return body + def indexes(self): """Return the collection indexes. @@ -1200,31 +1167,13 @@ def indexes(self): method='get', endpoint='/_api/index', params={'collection': self.name}, - command='db.{}.getIndexes()'.format(self.name), - read=self.name ) def response_handler(resp): if not resp.is_success: raise IndexListError(resp, request) - if self._is_transaction: - result = resp.body - else: - result = resp.body['indexes'] - - indexes = [] - for index in result: - index['id'] = index['id'].split('/', 1)[-1] - if 'minLength' in index: - index['min_length'] = index.pop('minLength') - if 'geoJson' in index: - index['geo_json'] = index.pop('geoJson') - if 'ignoreNull' in index: # pragma: no cover - index['ignore_none'] = index.pop('ignoreNull') - if 'selectivityEstimate' in index: - index['selectivity'] = index.pop('selectivityEstimate') - indexes.append(index) - return indexes + result = resp.body['indexes'] + return [self._format_index_details(index) for index in result] return self._execute(request, response_handler) @@ -1247,21 +1196,8 @@ def _add_index(self, data): def response_handler(resp): if not resp.is_success: raise IndexCreateError(resp, request) - details = resp.body - details['id'] = details['id'].split('/', 1)[1] - details.pop('error', None) - details.pop('code', None) - if 'minLength' in details: - details['min_length'] = details.pop('minLength') - if 'geoJson' in details: - details['geo_json'] = details.pop('geoJson') - if 'ignoreNull' in details: # pragma: no cover - details['ignore_none'] = details.pop('ignoreNull') - if 'selectivityEstimate' in details: - details['selectivity'] = details.pop('selectivityEstimate') - if 'isNewlyCreated' in details: - details['new'] = details.pop('isNewlyCreated') - return details + + return self._format_index_details(resp.body) return self._execute(request, response_handler) @@ -1269,7 +1205,9 @@ def add_hash_index(self, fields, unique=None, sparse=None, - deduplicate=None): + deduplicate=None, + name=None, + in_background=None): """Create a new hash index. :param fields: Document fields to index. @@ -1282,24 +1220,36 @@ def add_hash_index(self, :param deduplicate: If set to True, inserting duplicate index values from the same document triggers unique constraint errors. :type deduplicate: bool + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ data = {'type': 'hash', 'fields': fields} + if unique is not None: data['unique'] = unique if sparse is not None: data['sparse'] = sparse if deduplicate is not None: data['deduplicate'] = deduplicate + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + return self._add_index(data) def add_skiplist_index(self, fields, unique=None, sparse=None, - deduplicate=None): + deduplicate=None, + name=None, + in_background=None): """Create a new skiplist index. :param fields: Document fields to index. @@ -1312,20 +1262,34 @@ def add_skiplist_index(self, :param deduplicate: If set to True, inserting duplicate index values from the same document triggers unique constraint errors. :type deduplicate: bool + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ data = {'type': 'skiplist', 'fields': fields} + if unique is not None: data['unique'] = unique if sparse is not None: data['sparse'] = sparse if deduplicate is not None: data['deduplicate'] = deduplicate + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + return self._add_index(data) - def add_geo_index(self, fields, ordered=None): + def add_geo_index(self, + fields, + ordered=None, + name=None, + in_background=None): """Create a new geo-spatial index. :param fields: A single document field or a list of document fields. If @@ -1335,32 +1299,61 @@ def add_geo_index(self, fields, ordered=None): :type fields: str | unicode | list :param ordered: Whether the order is longitude, then latitude. :type ordered: bool + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ data = {'type': 'geo', 'fields': fields} + if ordered is not None: data['geoJson'] = ordered + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + return self._add_index(data) - def add_fulltext_index(self, fields, min_length=None): + def add_fulltext_index(self, + fields, + min_length=None, + name=None, + in_background=None): """Create a new fulltext index. :param fields: Document fields to index. :type fields: [str | unicode] :param min_length: Minimum number of characters to index. :type min_length: int + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ data = {'type': 'fulltext', 'fields': fields} + if min_length is not None: data['minLength'] = min_length + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + return self._add_index(data) - def add_persistent_index(self, fields, unique=None, sparse=None): + def add_persistent_index(self, + fields, + unique=None, + sparse=None, + name=None, + in_background=None): """Create a new persistent index. Unique persistent indexes on non-sharded keys are not supported in a @@ -1374,15 +1367,53 @@ def add_persistent_index(self, fields, unique=None, sparse=None): the indexed fields, or documents that have a value of None in any of the indexed fields. :type sparse: bool + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool :return: New index details. :rtype: dict :raise arango.exceptions.IndexCreateError: If create fails. """ data = {'type': 'persistent', 'fields': fields} + if unique is not None: data['unique'] = unique if sparse is not None: data['sparse'] = sparse + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + + return self._add_index(data) + + def add_ttl_index(self, + fields, + expiry_time, + name=None, + in_background=None): + """Create a new TTL (time-to-live) index. + + :param fields: Document field to index. + :type fields: [str | unicode] + :param expiry_time: Time of expiry in seconds after document creation. + :type expiry_time: int + :param name: Optional name for the index. + :type name: str | unicode + :param in_background: Do not hold the collection lock. + :type in_background: bool + :return: New index details. + :rtype: dict + :raise arango.exceptions.IndexCreateError: If create fails. + """ + data = {'type': 'ttl', 'fields': fields, 'expireAfter': expiry_time} + + if name is not None: + data['name'] = name + if in_background is not None: + data['inBackground'] = in_background + return self._add_index(data) def delete_index(self, index_id, ignore_missing=False): @@ -1472,16 +1503,10 @@ def get(self, document, rev=None, check_rev=True): """ handle, body, headers = self._prep_from_doc(document, rev, check_rev) - command = 'db.{}.exists({}) || undefined'.format( - self.name, - dumps(body) - ) if self._is_transaction else None - request = Request( method='get', endpoint='/_api/document/{}'.format(handle), headers=headers, - command=command, read=self.name ) @@ -1539,18 +1564,11 @@ def insert(self, if sync is not None: params['waitForSync'] = sync - command = 'db.{}.insert({},{})'.format( - self.name, - dumps(document), - dumps(params) - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/document/{}'.format(self.name), data=document, params=params, - command=command, write=self.name ) @@ -1611,19 +1629,11 @@ def insert_many(self, if sync is not None: params['waitForSync'] = sync - command = 'db.{}.insert({},{})'.format( - self.name, - dumps(documents), - dumps(params) - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/document/{}'.format(self.name), data=documents, - params=params, - command=command, - write=self.name + params=params ) def response_handler(resp): @@ -1703,12 +1713,6 @@ def update(self, if sync is not None: params['waitForSync'] = sync - command = 'db.{col}.update({doc},{doc},{opts})'.format( - col=self.name, - doc=dumps(document), - opts=dumps(params) - ) if self._is_transaction else None - request = Request( method='patch', endpoint='/_api/document/{}'.format( @@ -1716,7 +1720,6 @@ def update(self, ), data=document, params=params, - command=command, write=self.name ) @@ -1785,18 +1788,12 @@ def update_many(self, params['waitForSync'] = sync documents = [self._ensure_key_in_body(doc) for doc in documents] - command = 'db.{col}.update({docs},{docs},{opts})'.format( - col=self.name, - docs=dumps(documents), - opts=dumps(params) - ) if self._is_transaction else None request = Request( method='patch', endpoint='/_api/document/{}'.format(self.name), data=documents, params=params, - command=command, write=self.name ) @@ -1870,26 +1867,16 @@ def update_match(self, if sync is not None: data['waitForSync'] = sync - command = 'db.{}.updateByExample({},{},{})'.format( - self.name, - dumps(filters), - dumps(body), - dumps(data) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/update-by-example', data=data, - command=command, write=self.name ) def response_handler(resp): if not resp.is_success: raise DocumentUpdateError(resp, request) - if self._is_transaction: - return resp.body return resp.body['updated'] return self._execute(request, response_handler) @@ -1935,12 +1922,6 @@ def replace(self, if sync is not None: params['waitForSync'] = sync - command = 'db.{col}.replace({doc},{doc},{opts})'.format( - col=self.name, - doc=dumps(document), - opts=dumps(params) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/document/{}'.format( @@ -1948,7 +1929,6 @@ def replace(self, ), params=params, data=document, - command=command, write=self.name ) @@ -2008,18 +1988,12 @@ def replace_many(self, params['waitForSync'] = sync documents = [self._ensure_key_in_body(doc) for doc in documents] - command = 'db.{col}.replace({docs},{docs},{opts})'.format( - col=self.name, - docs=dumps(documents), - opts=dumps(params) - ) if self._is_transaction else None request = Request( method='put', endpoint='/_api/document/{}'.format(self.name), params=params, data=documents, - command=command, write=self.name ) @@ -2078,26 +2052,16 @@ def replace_match(self, filters, body, limit=None, sync=None): if sync is not None: data['waitForSync'] = sync - command = 'db.{}.replaceByExample({},{},{})'.format( - self.name, - dumps(filters), - dumps(body), - dumps(data) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/replace-by-example', data=data, - command=command, write=self.name ) def response_handler(resp): if not resp.is_success: raise DocumentReplaceError(resp, request) - if self._is_transaction: - return resp.body return resp.body['replaced'] return self._execute(request, response_handler) @@ -2151,18 +2115,11 @@ def delete(self, if sync is not None: params['waitForSync'] = sync - command = 'db.{}.remove({},{})'.format( - self.name, - dumps(body), - dumps(params) - ) if self._is_transaction else None - request = Request( method='delete', endpoint='/_api/document/{}'.format(handle), params=params, headers=headers, - command=command, write=self.name ) @@ -2219,18 +2176,12 @@ def delete_many(self, self._ensure_key_in_body(doc) if isinstance(doc, dict) else doc for doc in documents ] - command = 'db.{}.remove({},{})'.format( - self.name, - dumps(documents), - dumps(params) - ) if self._is_transaction else None request = Request( method='delete', endpoint='/_api/document/{}'.format(self.name), params=params, data=documents, - command=command, write=self.name ) @@ -2281,25 +2232,16 @@ def delete_match(self, filters, limit=None, sync=None): if limit is not None and limit != 0: data['limit'] = limit - command = 'db.{}.removeByExample({},{})'.format( - self.name, - dumps(filters), - dumps(data) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/simple/remove-by-example', data=data, - command=command, write=self.name ) def response_handler(resp): if not resp.is_success: raise DocumentDeleteError(resp, request) - if self._is_transaction: - return resp.body return resp.body['deleted'] return self._execute(request, response_handler) @@ -2383,7 +2325,8 @@ def import_bulk(self, method='post', endpoint='/_api/import', data=documents, - params=params + params=params, + write=self.name ) def response_handler(resp): @@ -2445,19 +2388,12 @@ def get(self, vertex, rev=None, check_rev=True): """ handle, body, headers = self._prep_from_doc(vertex, rev, check_rev) - command = 'gm._graph("{}").{}.document({})'.format( - self.graph, - self.name, - dumps(body) - ) if self._is_transaction else None - request = Request( method='get', endpoint='/_api/gharial/{}/vertex/{}'.format( self._graph, handle ), headers=headers, - command=command, read=self.name ) @@ -2468,8 +2404,6 @@ def response_handler(resp): raise DocumentRevisionError(resp, request) if not resp.is_success: raise DocumentGetError(resp, request) - if self._is_transaction: - return resp.body return resp.body['vertex'] return self._execute(request, response_handler) @@ -2497,13 +2431,6 @@ def insert(self, vertex, sync=None, silent=False): if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.save({},{})'.format( - self.graph, - self.name, - dumps(vertex), - dumps(params) - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/gharial/{}/vertex/{}'.format( @@ -2511,7 +2438,6 @@ def insert(self, vertex, sync=None, silent=False): ), data=vertex, params=params, - command=command, write=self.name ) @@ -2520,8 +2446,6 @@ def response_handler(resp): raise DocumentInsertError(resp, request) if silent is True: return True - if self._is_transaction: - return resp.body return resp.body['vertex'] return self._execute(request, response_handler) @@ -2550,6 +2474,12 @@ def update(self, :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param return_old: Include body of the old document in the returned + metadata. Ignored if parameter **silent** is set to True. + :type return_old: bool + :param return_new: Include body of the new document in the returned + metadata. Ignored if parameter **silent** is set to True. + :type return_new: bool :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2568,14 +2498,6 @@ def update(self, if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.update("{}",{},{})'.format( - self.graph, - self.name, - vertex_id, - dumps(vertex), - dumps(params) - ) if self._is_transaction else None - request = Request( method='patch', endpoint='/_api/gharial/{}/vertex/{}'.format( @@ -2584,7 +2506,6 @@ def update(self, headers=headers, params=params, data=vertex, - command=command, write=self.name ) @@ -2595,8 +2516,6 @@ def response_handler(resp): raise DocumentUpdateError(resp, request) if silent is True: return True - if self._is_transaction: - result = resp.body else: result = resp.body['vertex'] result['_old_rev'] = result.pop('_oldRev') @@ -2630,14 +2549,6 @@ def replace(self, vertex, check_rev=True, sync=None, silent=False): if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.replace("{}",{},{})'.format( - self.graph, - self.name, - vertex_id, - dumps(vertex), - dumps(params) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/gharial/{}/vertex/{}'.format( @@ -2646,7 +2557,6 @@ def replace(self, vertex, check_rev=True, sync=None, silent=False): headers=headers, params=params, data=vertex, - command=command, write=self.name ) @@ -2657,8 +2567,6 @@ def response_handler(resp): raise DocumentReplaceError(resp, request) if silent is True: return True - if self._is_transaction: - result = resp.body else: result = resp.body['vertex'] result['_old_rev'] = result.pop('_oldRev') @@ -2699,12 +2607,6 @@ def delete(self, handle, _, headers = self._prep_from_doc(vertex, rev, check_rev) params = {} if sync is None else {'waitForSync': sync} - command = 'gm._graph("{}").{}.remove("{}",{})'.format( - self.graph, - self.name, - handle, - dumps(params) - ) if self._is_transaction else None request = Request( method='delete', @@ -2713,7 +2615,6 @@ def delete(self, ), params=params, headers=headers, - command=command, write=self.name ) @@ -2780,19 +2681,12 @@ def get(self, edge, rev=None, check_rev=True): """ handle, body, headers = self._prep_from_doc(edge, rev, check_rev) - command = 'gm._graph("{}").{}.document({})'.format( - self.graph, - self.name, - dumps(body) - ) if self._is_transaction else None - request = Request( method='get', endpoint='/_api/gharial/{}/edge/{}'.format( self._graph, handle ), headers=headers, - command=command, read=self.name ) @@ -2803,8 +2697,6 @@ def response_handler(resp): raise DocumentRevisionError(resp, request) if not resp.is_success: raise DocumentGetError(resp, request) - if self._is_transaction: - return resp.body return resp.body['edge'] return self._execute(request, response_handler) @@ -2833,15 +2725,6 @@ def insert(self, edge, sync=None, silent=False): if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.save("{}","{}",{},{})'.format( - self.graph, - self.name, - edge['_from'], - edge['_to'], - dumps(edge), - dumps(params) - ) if self._is_transaction else None - request = Request( method='post', endpoint='/_api/gharial/{}/edge/{}'.format( @@ -2849,7 +2732,6 @@ def insert(self, edge, sync=None, silent=False): ), data=edge, params=params, - command=command, write=self.name ) @@ -2858,8 +2740,6 @@ def response_handler(resp): raise DocumentInsertError(resp, request) if silent is True: return True - if self._is_transaction: - return resp.body return resp.body['edge'] return self._execute(request, response_handler) @@ -2902,14 +2782,6 @@ def update(self, if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.update("{}",{},{})'.format( - self.graph, - self.name, - edge_id, - dumps(edge), - dumps(params) - ) if self._is_transaction else None - request = Request( method='patch', endpoint='/_api/gharial/{}/edge/{}'.format( @@ -2918,7 +2790,6 @@ def update(self, headers=headers, params=params, data=edge, - command=command, write=self.name ) @@ -2929,10 +2800,7 @@ def response_handler(resp): raise DocumentUpdateError(resp, request) if silent is True: return True - if self._is_transaction: - result = resp.body - else: - result = resp.body['edge'] + result = resp.body['edge'] result['_old_rev'] = result.pop('_oldRev') return result @@ -2965,14 +2833,6 @@ def replace(self, edge, check_rev=True, sync=None, silent=False): if sync is not None: params['waitForSync'] = sync - command = 'gm._graph("{}").{}.replace("{}",{},{})'.format( - self.graph, - self.name, - edge_id, - dumps(edge), - dumps(params) - ) if self._is_transaction else None - request = Request( method='put', endpoint='/_api/gharial/{}/edge/{}'.format( @@ -2981,7 +2841,6 @@ def replace(self, edge, check_rev=True, sync=None, silent=False): headers=headers, params=params, data=edge, - command=command, write=self.name ) @@ -2992,10 +2851,7 @@ def response_handler(resp): raise DocumentReplaceError(resp, request) if silent is True: return True - if self._is_transaction: - result = resp.body - else: - result = resp.body['edge'] + result = resp.body['edge'] result['_old_rev'] = result.pop('_oldRev') return result @@ -3034,12 +2890,6 @@ def delete(self, handle, _, headers = self._prep_from_doc(edge, rev, check_rev) params = {} if sync is None else {'waitForSync': sync} - command = 'gm._graph("{}").{}.remove("{}",{})'.format( - self.graph, - self.name, - handle, - dumps(params) - ) if self._is_transaction else None request = Request( method='delete', @@ -3048,7 +2898,6 @@ def delete(self, ), params=params, headers=headers, - command=command, write=self.name ) @@ -3111,7 +2960,8 @@ def edges(self, vertex, direction=None): request = Request( method='get', endpoint='/_api/edges/{}'.format(self.name), - params=params + params=params, + read=self.name ) def response_handler(resp): diff --git a/arango/connection.py b/arango/connection.py index 2464987d..915041f2 100644 --- a/arango/connection.py +++ b/arango/connection.py @@ -1,17 +1,23 @@ from __future__ import absolute_import, unicode_literals -from arango.http import DefaultHTTPClient +from six import string_types + +from arango.exceptions import ServerConnectionError __all__ = ['Connection'] class Connection(object): - """HTTP connection to specific ArangoDB database. + """Connection to specific ArangoDB database. - :param url: ArangoDB base URL. - :type url: str | unicode - :param db: Database name. - :type db: str | unicode + :param hosts: Host URL or list of URLs (coordinators in a cluster). + :type hosts: [str | unicode] + :param host_resolver: Host resolver (used for clusters). + :type host_resolver: arango.resolver.HostResolver + :param sessions: HTTP session objects per host. + :type sessions: [requests.Session] + :param db_name: Database name. + :type db_name: str | unicode :param username: Username. :type username: str | unicode :param password: Password. @@ -20,21 +26,25 @@ class Connection(object): :type http_client: arango.http.HTTPClient """ - def __init__(self, url, db, username, password, http_client): - self._url_prefix = '{}/_db/{}'.format(url, db) - self._db_name = db + def __init__(self, + hosts, + host_resolver, + sessions, + db_name, + username, + password, + http_client, + serializer, + deserializer): + self._url_prefixes = ['{}/_db/{}'.format(h, db_name) for h in hosts] + self._host_resolver = host_resolver + self._sessions = sessions + self._db_name = db_name self._username = username self._auth = (username, password) - self._http_client = http_client or DefaultHTTPClient() - - @property - def url_prefix(self): - """Return the ArangoDB URL prefix (base URL + database name). - - :returns: ArangoDB URL prefix. - :rtype: str | unicode - """ - return self._url_prefix + self._http = http_client + self._serializer = serializer + self._deserializer = deserializer @property def username(self): @@ -54,6 +64,47 @@ def db_name(self): """ return self._db_name + def serialize(self, obj): + """Serialize the object and return the string. + + :param obj: Object to serialize. + :type obj: str | unicode | bool | int | list | dict + :return: Serialized string. + :rtype: str | unicode + """ + return self._serializer(obj) + + def deserialize(self, string): + """De-serialize the string and return the object. + + :param string: String to de-serialize. + :type string: str | unicode + :return: De-serialized object. + :rtype: str | unicode | bool | int | list | dict + """ + return self._deserializer(string) + + def prep_response(self, response): + """Populate the response with details and return it. + + :param response: HTTP response. + :type response: arango.response.Response + :return: HTTP response. + :rtype: arango.response.Response + """ + try: + response.body = self.deserialize(response.raw_body) + except (ValueError, TypeError): + response.body = response.raw_body + else: + if isinstance(response.body, dict): + response.error_code = response.body.get('errorNum') + response.error_message = response.body.get('errorMessage') + + http_ok = 200 <= response.status_code < 300 + response.is_success = http_ok and response.error_code is None + return response + def send_request(self, request): """Send an HTTP request to ArangoDB server. @@ -62,11 +113,37 @@ def send_request(self, request): :return: HTTP response. :rtype: arango.response.Response """ - return self._http_client.send_request( + if request.data is None or isinstance(request.data, string_types): + normalized_data = request.data + else: + normalized_data = self.serialize(request.data) + + host_index = self._host_resolver.get_host_index() + response = self._http.send_request( + session=self._sessions[host_index], method=request.method, - url=self._url_prefix + request.endpoint, + url=self._url_prefixes[host_index] + request.endpoint, params=request.params, - data=request.data, + data=normalized_data, headers=request.headers, auth=self._auth, ) + return self.prep_response(response) + + def ping(self): + for host_index in range(len(self._sessions)): + resp = self._http.send_request( + session=self._sessions[host_index], + method='get', + url=self._url_prefixes[host_index] + '/_api/collection', + auth=self._auth, + ) + resp = self.prep_response(resp) + + code = resp.status_code + if code in {401, 403}: + raise ServerConnectionError('bad username and/or password') + if not (200 <= code < 300): # pragma: no cover + raise ServerConnectionError( + resp.error_message or 'bad server response') + return code diff --git a/arango/cursor.py b/arango/cursor.py index 2b08f0c6..e68729e6 100644 --- a/arango/cursor.py +++ b/arango/cursor.py @@ -20,10 +20,6 @@ class Cursor(object): are *stateful* as they store the fetched items in-memory. They must not be shared across threads without proper locking mechanism. - In transactions, the entire result set is loaded into the cursor. Therefore - you must be mindful of client-side memory capacity when running queries - that can potentially return a large result set. - :param connection: HTTP connection. :type connection: arango.connection.Connection :param init_data: Cursor initialization data. @@ -56,17 +52,7 @@ def __init__(self, connection, init_data, cursor_type='cursor'): self._stats = None self._profile = None self._warnings = None - - if isinstance(init_data, list): - # In transactions, cursor initialization data is a list containing - # the entire result set. - self._has_more = False - self._batch.extend(init_data) - self._count = len(init_data) - else: - # In other execution contexts, cursor initialization data is a dict - # containing cursor metadata (e.g. ID, parameters). - self._update(init_data) + self._update(init_data) def __iter__(self): return self @@ -285,7 +271,7 @@ def close(self, ignore_missing=False): :return: True if cursor was closed successfully, False if cursor was missing on the server and **ignore_missing** was set to True, None if there are no cursors to close server-side (e.g. result set is - smaller than the batch size, or in transactions). + smaller than the batch size). :rtype: bool | None :raise arango.exceptions.CursorCloseError: If operation fails. :raise arango.exceptions.CursorStateError: If cursor ID is not set. diff --git a/arango/database.py b/arango/database.py index a74a0f6f..998ed727 100644 --- a/arango/database.py +++ b/arango/database.py @@ -21,6 +21,10 @@ ) from arango.collection import StandardCollection from arango.exceptions import ( + AnalyzerCreateError, + AnalyzerDeleteError, + AnalyzerGetError, + AnalyzerListError, AsyncJobClearError, AsyncJobListError, CollectionCreateError, @@ -37,7 +41,6 @@ PermissionGetError, PermissionResetError, PermissionUpdateError, - ServerConnectionError, ServerEndpointsError, ServerEngineError, ServerDetailsError, @@ -381,8 +384,7 @@ def engine(self): """ request = Request( method='get', - endpoint='/_api/engine', - command='db._engine()' + endpoint='/_api/engine' ) def response_handler(resp): @@ -392,29 +394,6 @@ def response_handler(resp): return self._execute(request, response_handler) - def ping(self): - """Ping the ArangoDB server by sending a test request. - - :return: Response code from server. - :rtype: int - :raise arango.exceptions.ServerConnectionError: If ping fails. - """ - request = Request( - method='get', - endpoint='/_api/collection', - ) - - def response_handler(resp): - code = resp.status_code - if code in {401, 403}: - raise ServerConnectionError('bad username and/or password') - if not resp.is_success: # pragma: no cover - raise ServerConnectionError( - resp.error_message or 'bad server response') - return code - - return self._execute(request, response_handler) - def statistics(self, description=False): """Return server statistics. @@ -672,7 +651,7 @@ def reload_routing(self): def response_handler(resp): if not resp.is_success: raise ServerReloadRoutingError(resp, request) - return 'error' not in resp.body + return True return self._execute(request, response_handler) @@ -849,7 +828,9 @@ def create_collection(self, replication_factor=None, shard_like=None, sync_replication=None, - enforce_replication_factor=None): + enforce_replication_factor=None, + sharding_strategy=None, + smart_join_attribute=None): """Create a new collection. :param name: Collection name. @@ -917,6 +898,21 @@ def create_collection(self, :param enforce_replication_factor: Check if there are enough replicas available at creation time, or halt the operation. :type enforce_replication_factor: bool + :param sharding_strategy: Sharding strategy. Available for ArangoDB + version 3.4 and up only. Possible values are "community-compat", + "enterprise-compat", "enterprise-smart-edge-compat", "hash" and + "enterprise-hash-smart-edge". Refer to ArangoDB documentation for + more details on each value. + :type sharding_strategy: str | unicode + :param smart_join_attribute: Attribute of the collection which must + contain the shard key value of the smart join collection. The shard + key for the documents must contain the value of this attribute, + followed by a colon ":" and the primary key of the document. + Requires parameter **shard_like** to be set to the name of another + collection, and parameter **shard_fields** to be set to a single + shard key attribute, with another colon ":" at the end. Available + only for enterprise version of ArangoDB. + :type smart_join_attribute: str | unicode :return: Standard collection API wrapper. :rtype: arango.collection.StandardCollection :raise arango.exceptions.CollectionCreateError: If create fails. @@ -948,6 +944,10 @@ def create_collection(self, data['replicationFactor'] = replication_factor if shard_like is not None: data['distributeShardsLike'] = shard_like + if sharding_strategy is not None: + data['shardingStrategy'] = sharding_strategy + if smart_join_attribute is not None: + data['smartJoinAttribute'] = smart_join_attribute params = {} if sync_replication is not None: @@ -2035,9 +2035,11 @@ def delete_view(self, name, ignore_missing=False): :param name: View name. :type name: str | unicode + :param ignore_missing: Do not raise an exception on missing view. + :type ignore_missing: bool :return: True if view was deleted successfully, False if view was not found and **ignore_missing** was set to True. - :rtype: dict + :rtype: bool :raise arango.exceptions.ViewDeleteError: If delete fails. """ request = Request( @@ -2059,6 +2061,8 @@ def rename_view(self, name, new_name): :param name: View name. :type name: str | unicode + :param new_name: New view name. + :type new_name: str | unicode :return: View details. :rtype: dict :raise arango.exceptions.ViewRenameError: If delete fails. @@ -2076,6 +2080,204 @@ def response_handler(resp): return self._execute(request, response_handler) + ################################ + # ArangoSearch View Management # + ################################ + + def create_arangosearch_view(self, name, properties=None): + """Create an ArangoSearch view. + + :param name: View name. + :type name: str | unicode + :param properties: View properties. + :type properties: dict + :return: View details. + :rtype: dict + :raise arango.exceptions.ViewCreateError: If create fails. + """ + data = {'name': name, 'type': 'arangosearch'} + + if properties is not None: + data.update(properties) + + request = Request( + method='post', + endpoint='/_api/view#ArangoSearch', + data=data + ) + + def response_handler(resp): + if resp.is_success: + return resp.body + raise ViewCreateError(resp, request) + + return self._execute(request, response_handler) + + def update_arangosearch_view(self, name, properties): + """Update an ArangoSearch view. + + :param name: View name. + :type name: str | unicode + :param properties: View properties. + :type properties: dict + :return: View details. + :rtype: dict + :raise arango.exceptions.ViewUpdateError: If update fails. + """ + request = Request( + method='patch', + endpoint='/_api/view/{}/properties#ArangoSearch'.format(name), + data=properties + ) + + def response_handler(resp): + if resp.is_success: + return resp.body + raise ViewUpdateError(resp, request) + + return self._execute(request, response_handler) + + def replace_arangosearch_view(self, name, properties): + """Replace an ArangoSearch view. + + :param name: View name. + :type name: str | unicode + :param properties: View properties. + :type properties: dict + :return: View details. + :rtype: dict + :raise arango.exceptions.ViewReplaceError: If replace fails. + """ + request = Request( + method='put', + endpoint='/_api/view/{}/properties#ArangoSearch'.format(name), + data=properties + ) + + def response_handler(resp): + if resp.is_success: + return resp.body + raise ViewReplaceError(resp, request) + + return self._execute(request, response_handler) + + ####################### + # Analyzer Management # + ####################### + + def analyzers(self): + """Return list of analyzers. + + :return: List of analyzers. + :rtype: [dict] + :raise arango.exceptions.AnalyzerListError: If retrieval fails. + """ + request = Request( + method='get', + endpoint='/_api/analyzer' + ) + + def response_handler(resp): + if resp.is_success: + resp.body.pop('error') + resp.body.pop('code') + return resp.body['result'] + raise AnalyzerListError(resp, request) + + return self._execute(request, response_handler) + + def analyzer(self, name): + """Return analyzer details. + + :param name: Analyzer name. + :type name: str | unicode + :return: Analyzer details. + :rtype: dict + :raise arango.exceptions.AnalyzerGetError: If retrieval fails. + """ + request = Request( + method='get', + endpoint='/_api/analyzer/{}'.format(name) + ) + + def response_handler(resp): + if resp.is_success: + resp.body.pop('error') + resp.body.pop('code') + return resp.body + raise AnalyzerGetError(resp, request) + + return self._execute(request, response_handler) + + def create_analyzer(self, + name, + analyzer_type, + properties=None, + features=None): + """Create an analyzer. + + :param name: Analyzer name. + :type name: str | unicode + :param analyzer_type: Analyzer type. + :type analyzer_type: str | unicode + :param properties: Analyzer properties. + :type properties: dict + :param features: Analyzer features. + :type features: list + :return: Analyzer details. + :rtype: dict + :raise arango.exceptions.AnalyzerCreateError: If create fails. + """ + data = {'name': name, 'type': analyzer_type} + + if properties is not None: + data['properties'] = properties + + if features is not None: + data['features'] = features + + request = Request( + method='post', + endpoint='/_api/analyzer', + data=data + ) + + def response_handler(resp): + if resp.is_success: + return resp.body + raise AnalyzerCreateError(resp, request) + + return self._execute(request, response_handler) + + def delete_analyzer(self, name, force=False, ignore_missing=False): + """Delete an analyzer. + + :param name: Analyzer name. + :type name: str | unicode + :param force: Remove the analyzer configuration even if in use. + :type force: bool + :param ignore_missing: Do not raise an exception on missing analyzer. + :type ignore_missing: bool + :return: True if analyzer was deleted successfully, False if analyzer + was not found and **ignore_missing** was set to True. + :rtype: bool + :raise arango.exceptions.AnalyzerDeleteError: If delete fails. + """ + request = Request( + method='delete', + endpoint='/_api/analyzer/{}'.format(name), + params={'force': force} + ) + + def response_handler(resp): + if resp.error_code in {1202, 404} and ignore_missing: + return False + if resp.is_success: + return True + raise AnalyzerDeleteError(resp, request) + + return self._execute(request, response_handler) + class StandardDatabase(Database): """Standard database API wrapper. @@ -2101,7 +2303,7 @@ def begin_async_execution(self, return_result=True): results from server once available. If set to False, API executions return None and no results are stored on server. :type return_result: bool - :return: Database API wrapper built specifically for async execution. + :return: Database API wrapper object specifically for async execution. :rtype: arango.database.AsyncDatabase """ return AsyncDatabase(self._conn, return_result) @@ -2114,46 +2316,54 @@ def begin_batch_execution(self, return_result=True): commit. If set to False, API executions return None and no results are tracked client-side. :type return_result: bool - :return: Database API wrapper built specifically for batch execution. + :return: Database API wrapper object specifically for batch execution. :rtype: arango.database.BatchDatabase """ return BatchDatabase(self._conn, return_result) def begin_transaction(self, - return_result=True, - timeout=None, - sync=None, read=None, - write=None): - """Begin transaction. - - :param return_result: If set to True, API executions return instances - of :class:`arango.job.TransactionJob` that are populated with - results on commit. If set to False, API executions return None and - no results are tracked client-side. - :type return_result: bool - :param read: Names of collections read during transaction. If not - specified, they are added automatically as jobs are queued. - :type read: [str | unicode] - :param write: Names of collections written to during transaction. - If not specified, they are added automatically as jobs are queued. - :type write: [str | unicode] - :param timeout: Timeout for waiting on collection locks. If set to 0, - ArangoDB server waits indefinitely. If not set, system default - value is used. - :type timeout: int - :param sync: Block until the transaction is synchronized to disk. + write=None, + exclusive=None, + sync=None, + allow_implicit=None, + lock_timeout=None, + max_size=None): + """Begin a transaction. + + :param read: Name(s) of collections read during transaction. Read-only + collections are added lazily but should be declared if possible to + avoid deadlocks. + :type read: str | unicode | [str | unicode] + :param write: Name(s) of collections written to during transaction with + shared access. + :type write: str | unicode | [str | unicode] + :param exclusive: Name(s) of collections written to during transaction + with exclusive access. + :type exclusive: str | unicode | [str | unicode] + :param sync: Block until operation is synchronized to disk. :type sync: bool - :return: Database API wrapper built specifically for transactions. + :param allow_implicit: Allow reading from undeclared collections. + :type allow_implicit: bool + :param lock_timeout: Timeout for waiting on collection locks. If not + given, a default value is used. Setting it to 0 disables the + timeout. + :type lock_timeout: int + :param max_size: Max transaction size in bytes. Applicable to RocksDB + storage engine only. + :type max_size: + :return: Database API wrapper object specifically for transactions. :rtype: arango.database.TransactionDatabase """ return TransactionDatabase( connection=self._conn, - return_result=return_result, read=read, write=write, - timeout=timeout, - sync=sync + exclusive=exclusive, + sync=sync, + allow_implicit=allow_implicit, + lock_timeout=lock_timeout, + max_size=max_size ) @@ -2244,67 +2454,86 @@ class TransactionDatabase(Database): :param connection: HTTP connection. :type connection: arango.connection.Connection - :param return_result: If set to True, API executions return instances of - :class:`arango.job.TransactionJob` that are populated with results on - commit. If set to False, API executions return None and no results are - tracked client-side. - :type return_result: bool - :param read: Names of collections read during transaction. - :type read: [str | unicode] - :param write: Names of collections written to during transaction. - :type write: [str | unicode] - :param timeout: Timeout for waiting on collection locks. If set to 0, the - ArangoDB server waits indefinitely. If not set, system default value - is used. - :type timeout: int + :param read: Name(s) of collections read during transaction. Read-only + collections are added lazily but should be declared if possible to + avoid deadlocks. + :type read: str | unicode | [str | unicode] + :param write: Name(s) of collections written to during transaction with + shared access. + :type write: str | unicode | [str | unicode] + :param exclusive: Name(s) of collections written to during transaction + with exclusive access. + :type exclusive: str | unicode | [str | unicode] :param sync: Block until operation is synchronized to disk. :type sync: bool + :param allow_implicit: Allow reading from undeclared collections. + :type allow_implicit: bool + :param lock_timeout: Timeout for waiting on collection locks. If not given, + a default value is used. Setting it to 0 disables the timeout. + :type lock_timeout: int + :param max_size: Max transaction size in bytes. Applicable to RocksDB + storage engine only. + :type max_size: int """ - def __init__(self, connection, return_result, read, write, timeout, sync): + def __init__(self, + connection, + read=None, + write=None, + exclusive=None, + sync=None, + allow_implicit=None, + lock_timeout=None, + max_size=None): super(TransactionDatabase, self).__init__( connection=connection, executor=TransactionExecutor( connection=connection, - return_result=return_result, read=read, write=write, - timeout=timeout, - sync=sync + exclusive=exclusive, + sync=sync, + allow_implicit=allow_implicit, + lock_timeout=lock_timeout, + max_size=max_size ) ) def __repr__(self): return ''.format(self.name) - def __enter__(self): - return self + @property + def transaction_id(self): + """Return the transaction ID. - def __exit__(self, exception, *_): - if exception is None: - self._executor.commit() + :return: Transaction ID. + :rtype: str | unicode + """ + return self._executor.id - def queued_jobs(self): - """Return the queued transaction jobs. + def transaction_status(self): + """Return the transaction status. - :return: Queued transaction jobs, or None if **return_result** was set - to False during initialization. - :rtype: [arango.job.TransactionJob] | None + :return: Transaction status. + :rtype: str | unicode + :raise arango.exceptions.TransactionStatusError: If retrieval fails. """ - return self._executor.jobs - - def commit(self): - """Execute the queued requests in a single transaction API request. + return self._executor.status() - If **return_result** parameter was set to True during initialization, - :class:`arango.job.TransactionJob` instances are populated with - results. + def commit_transaction(self): + """Commit the transaction. - :return: Transaction jobs, or None if **return_result** parameter was - set to False during initialization. - :rtype: [arango.job.TransactionJob] | None - :raise arango.exceptions.TransactionStateError: If the transaction was - already committed. - :raise arango.exceptions.TransactionExecuteError: If commit fails. + :return: True if commit was successful. + :rtype: bool + :raise arango.exceptions.TransactionCommitError: If commit fails. """ return self._executor.commit() + + def abort_transaction(self): + """Abort the transaction. + + :return: True if the abort operation was successful. + :rtype: bool + :raise arango.exceptions.TransactionAbortError: If abort fails. + """ + return self._executor.abort() diff --git a/arango/exceptions.py b/arango/exceptions.py index 55204115..ba4883e1 100644 --- a/arango/exceptions.py +++ b/arango/exceptions.py @@ -249,6 +249,14 @@ class CollectionRotateJournalError(ArangoServerError): """Failed to rotate collection journal.""" +class CollectionRecalculateCountError(ArangoServerError): + """Failed to recalculate document count.""" + + +class CollectionResponsibleShardError(ArangoServerError): + """Failed to retrieve responsible shard.""" + + ##################### # Cursor Exceptions # ##################### @@ -623,16 +631,24 @@ class TaskDeleteError(ArangoServerError): ########################## -class TransactionStateError(ArangoClientError): - """The transaction object was in bad state.""" +class TransactionExecuteError(ArangoServerError): + """Failed to execute raw transaction.""" -class TransactionJobResultError(ArangoClientError): - """Failed to retrieve transaction job result.""" +class TransactionInitError(ArangoServerError): + """Failed to initialize transaction.""" -class TransactionExecuteError(ArangoServerError): - """Failed to execute transaction API request""" +class TransactionStatusError(ArangoServerError): + """Failed to retrieve transaction status.""" + + +class TransactionCommitError(ArangoServerError): + """Failed to commit transaction.""" + + +class TransactionAbortError(ArangoServerError): + """Failed to abort transaction.""" ################### @@ -697,6 +713,26 @@ class ViewRenameError(ArangoServerError): """Failed to rename view.""" +####################### +# Analyzer Exceptions # +####################### + +class AnalyzerListError(ArangoServerError): + """Failed to retrieve analyzers.""" + + +class AnalyzerGetError(ArangoServerError): + """Failed to retrieve analyzer details.""" + + +class AnalyzerCreateError(ArangoServerError): + """Failed to create analyzer.""" + + +class AnalyzerDeleteError(ArangoServerError): + """Failed to delete analyzer.""" + + ######################### # Permission Exceptions # ######################### diff --git a/arango/executor.py b/arango/executor.py index 31de5f7f..6c9b2033 100644 --- a/arango/executor.py +++ b/arango/executor.py @@ -1,7 +1,5 @@ from __future__ import absolute_import, unicode_literals -from six import string_types - __all__ = [ 'DefaultExecutor', 'AsyncExecutor', @@ -12,17 +10,20 @@ from collections import OrderedDict from uuid import uuid4 +from six import moves + from arango.exceptions import ( AsyncExecuteError, BatchStateError, BatchExecuteError, - TransactionStateError, - TransactionExecuteError, + TransactionAbortError, + TransactionCommitError, + TransactionStatusError, + TransactionInitError ) from arango.job import ( AsyncJob, - BatchJob, - TransactionJob + BatchJob ) from arango.request import Request from arango.response import Response @@ -77,8 +78,8 @@ def execute(self, request, response_handler): :return: API execution result. :rtype: str | unicode | bool | int | list | dict """ - response = self._conn.send_request(request) - return response_handler(response) + resp = self._conn.send_request(request) + return response_handler(resp) class AsyncExecutor(Executor): @@ -143,6 +144,24 @@ def __init__(self, connection, return_result): self._queue = OrderedDict() self._committed = False + # noinspection PyMethodMayBeStatic,PyUnresolvedReferences + def _stringify_request(self, request): + path = request.endpoint + + if request.params is not None: + path += '?' + moves.urllib.parse.urlencode(request.params) + buffer = ['{} {} HTTP/1.1'.format(request.method, path)] + + if request.headers is not None: + for key, value in sorted(request.headers.items()): + buffer.append('{}: {}'.format(key, value)) + + if request.data is not None: + serialized = self._conn.serialize(request.data) + buffer.append('\r\n' + serialized) + + return '\r\n'.join(buffer) + @property def jobs(self): """Return the queued batch jobs. @@ -191,8 +210,8 @@ def commit(self): """ if self._committed: raise BatchStateError('batch already committed') - - self._committed = True + else: + self._committed = True if len(self._queue) == 0: return self.jobs @@ -200,13 +219,13 @@ def commit(self): # Boundary used for multipart request boundary = uuid4().hex - # Buffer for building the batch request payload + # Build the batch request payload buffer = [] for req, job in self._queue.values(): buffer.append('--{}'.format(boundary)) buffer.append('Content-Type: application/x-arango-batchpart') buffer.append('Content-Id: {}'.format(job.id)) - buffer.append('\r\n{}'.format(req)) + buffer.append('\r\n' + self._stringify_request(req)) buffer.append('--{}--'.format(boundary)) request = Request( @@ -216,7 +235,7 @@ def commit(self): 'Content-Type': 'multipart/form-data; boundary={}'.format(boundary) }, - data='\r\n'.join(buffer) + data='\r\n'.join(buffer), ) with suppress_warning('requests.packages.urllib3.connectionpool'): resp = self._conn.send_request(request) @@ -227,7 +246,9 @@ def commit(self): if not self._return_result: return None + url_prefix = resp.url.strip('/_api/batch') raw_resps = resp.raw_body.split('--{}'.format(boundary))[1:-1] + if len(self._queue) != len(raw_resps): raise BatchStateError( 'expecting {} parts in batch response but got {}' @@ -244,15 +265,17 @@ def commit(self): # Update the corresponding batch job queued_req, queued_job = self._queue[job_id] - queued_job._response = Response( + + queued_job._status = 'done' + resp = Response( method=queued_req.method, - url=self._conn.url_prefix + queued_req.endpoint, + url=url_prefix + queued_req.endpoint, headers={}, status_code=int(status_code), status_text=status_text, raw_body=raw_body ) - queued_job._status = 'done' + queued_job._response = self._conn.prep_response(resp) return self.jobs @@ -262,158 +285,140 @@ class TransactionExecutor(Executor): :param connection: HTTP connection. :type connection: arango.connection.Connection - :param return_result: If set to True, API executions return instances of - :class:`arango.job.TransactionJob` that are populated with results on - commit. If set to False, API executions return None and no results are - tracked client-side. - :type return_result: bool - :param timeout: Timeout for waiting on collection locks. If set to 0, - ArangoDB server waits indefinitely. If not set, system default value - is used. - :type timeout: int + :param read: Name(s) of collections read during transaction. Read-only + collections are added lazily but should be declared if possible to + avoid deadlocks. + :type read: str | unicode | [str | unicode] + :param write: Name(s) of collections written to during transaction with + shared access. + :type write: str | unicode | [str | unicode] + :param exclusive: Name(s) of collections written to during transaction + with exclusive access. + :type exclusive: str | unicode | [str | unicode] :param sync: Block until operation is synchronized to disk. :type sync: bool - :param read: Names of collections read during transaction. - :type read: [str | unicode] - :param write: Names of collections written to during transaction. - :type write: [str | unicode] + :param allow_implicit: Allow reading from undeclared collections. + :type allow_implicit: bool + :param lock_timeout: Timeout for waiting on collection locks. If not given, + a default value is used. Setting it to 0 disables the timeout. + :type lock_timeout: int + :param max_size: Max transaction size in bytes. Applicable to RocksDB + storage engine only. + :type max_size: int """ context = 'transaction' - def __init__(self, connection, return_result, read, write, timeout, sync): + def __init__(self, + connection, + read=None, + write=None, + exclusive=None, + sync=None, + allow_implicit=None, + lock_timeout=None, + max_size=None): super(TransactionExecutor, self).__init__(connection) - self._return_result = return_result - self._read = read - self._write = write - self._timeout = timeout - self._sync = sync - self._queue = OrderedDict() - self._committed = False + + collections = {} + if read is not None: + collections['read'] = read + if write is not None: + collections['write'] = write + if exclusive is not None: + collections['exclusive'] = exclusive + + data = {'collections': collections} + if sync is not None: + data['waitForSync'] = sync + if allow_implicit is not None: + data['allowImplicit'] = allow_implicit + if lock_timeout is not None: + data['lockTimeout'] = lock_timeout + if max_size is not None: + data['maxTransactionSize'] = max_size + + request = Request( + method='post', + endpoint='/_api/transaction/begin', + data=data + ) + resp = self._conn.send_request(request) + + if not resp.is_success: + raise TransactionInitError(resp, request) + + self._id = resp.body['result']['id'] @property - def jobs(self): - """Return the queued transaction jobs. + def id(self): + """Return the transaction ID. - :return: Transaction jobs or None if **return_result** parameter was - set to False during initialization. - :rtype: [arango.job.TransactionJob] | None + :return: Transaction ID. + :rtype: str | unicode """ - if not self._return_result: - return None - return [job for _, job in self._queue.values()] + return self._id def execute(self, request, response_handler): - """Place the request in the transaction queue. + """Execute API request in a transaction and return the result. :param request: HTTP request. :type request: arango.request.Request :param response_handler: HTTP response handler. :type response_handler: callable - :return: Transaction job or None if **return_result** parameter was - set to False during initialization. - :rtype: arango.job.TransactionJob | None - :raise arango.exceptions.TransactionStateError: If the transaction was - already committed or if the action does not support transactions. + :return: API execution result. + :rtype: str | unicode | bool | int | list | dict """ - if self._committed: - raise TransactionStateError('transaction already committed') - if request.command is None: - raise TransactionStateError('action not allowed in transaction') - - job = TransactionJob(response_handler) - self._queue[job.id] = (request, job) - return job if self._return_result else None + request.headers['x-arango-trx-id'] = self._id + resp = self._conn.send_request(request) + return response_handler(resp) - def commit(self): - """Execute the queued requests in a single transaction API request. + def status(self): + """Return the transaction status. - If **return_result** parameter was set to True during initialization, - :class:`arango.job.TransactionJob` instances are populated with - results. - - :return: Transaction jobs or None if **return_result** parameter was - set to False during initialization. - :rtype: [arango.job.TransactionJob] | None - :raise arango.exceptions.TransactionStateError: If the transaction was - already committed. - :raise arango.exceptions.TransactionExecuteError: If commit fails. + :return: Transaction status. + :rtype: str | unicode + :raise arango.exceptions.TransactionStatusError: If retrieval fails. """ - if self._committed: - raise TransactionStateError('transaction already committed') - - self._committed = True + request = Request( + method='get', + endpoint='/_api/transaction/{}'.format(self._id), + ) + resp = self._conn.send_request(request) - if len(self._queue) == 0: - return self.jobs + if resp.is_success: + return resp.body['result']['status'] + raise TransactionStatusError(resp, request) - write_collections = set() - if isinstance(self._write, string_types): - write_collections.add(self._write) - elif self._write is not None: - write_collections |= set(self._write) - - read_collections = set() - if isinstance(self._read, string_types): - read_collections.add(self._read) - elif self._read is not None: - read_collections |= set(self._read) - - # Buffer for building the transaction javascript command - cmd_buffer = [ - 'var db = require("internal").db', - 'var gm = require("@arangodb/general-graph")', - 'var result = {}' - ] - for req, job in self._queue.values(): - if isinstance(req.read, string_types): - read_collections.add(req.read) - elif req.read is not None: - read_collections |= set(req.read) - - if isinstance(req.write, string_types): - write_collections.add(req.write) - elif req.write is not None: - write_collections |= set(req.write) - - cmd_buffer.append('result["{}"] = {}'.format(job.id, req.command)) - - cmd_buffer.append('return result;') - - data = { - 'action': 'function () {{ {} }}'.format(';'.join(cmd_buffer)), - 'collections': { - 'read': list(read_collections), - 'write': list(write_collections), - 'allowImplicit': True - } - } - if self._timeout is not None: - data['lockTimeout'] = self._timeout - if self._sync is not None: - data['waitForSync'] = self._sync + def commit(self): + """Commit the transaction. + :return: True if commit was successful. + :rtype: bool + :raise arango.exceptions.TransactionCommitError: If commit fails. + """ request = Request( - method='post', - endpoint='/_api/transaction', - data=data, + method='put', + endpoint='/_api/transaction/{}'.format(self._id), ) resp = self._conn.send_request(request) - if not resp.is_success: - raise TransactionExecuteError(resp, request) + if resp.is_success: + return True + raise TransactionCommitError(resp, request) - if not self._return_result: - return None + def abort(self): + """Abort the transaction. - result = resp.body['result'] - for req, job in self._queue.values(): - job._response = Response( - method=req.method, - url=self._conn.url_prefix + req.endpoint, - headers={}, - status_code=200, - status_text='OK', - raw_body=result.get(job.id) - ) - job._status = 'done' - return self.jobs + :return: True if the abort operation was successful. + :rtype: bool + :raise arango.exceptions.TransactionAbortError: If abort fails. + """ + request = Request( + method='delete', + endpoint='/_api/transaction/{}'.format(self._id), + ) + resp = self._conn.send_request(request) + + if resp.is_success: + return True + raise TransactionAbortError(resp, request) diff --git a/arango/foxx.py b/arango/foxx.py index 2393387e..2f7dbe03 100644 --- a/arango/foxx.py +++ b/arango/foxx.py @@ -654,7 +654,8 @@ def run_tests(self, mount, reporter='default', idiomatic=None, - output_format=None): + output_format=None, + name_filter=None): """Run service tests. :param mount: Service mount path (e.g "/_admin/aardvark"). @@ -674,6 +675,9 @@ def run_tests(self, text TAP report. When using "xunit" reporter, settings this to "xml" returns an XML instead of JSONML. :type output_format: str | unicode + :param name_filter: Only run tests whose full name (test suite and + test case) matches the given string. + :type name_filter: str | unicode :return: Reporter output (e.g. raw JSON string, XML, plain text). :rtype: str | unicode :raise arango.exceptions.FoxxTestRunError: If test fails. @@ -681,6 +685,8 @@ def run_tests(self, params = {'mount': mount, 'reporter': reporter} if idiomatic is not None: params['idiomatic'] = idiomatic + if name_filter is not None: + params['filter'] = name_filter headers = {} if output_format == 'x-ldjson': diff --git a/arango/http.py b/arango/http.py index 498f5a24..708841d3 100644 --- a/arango/http.py +++ b/arango/http.py @@ -14,8 +14,22 @@ class HTTPClient(object): # pragma: no cover __metaclass__ = ABCMeta + @abstractmethod + def create_session(self, host): + """Return a new requests session given the host URL. + + This method must be overridden by the user. + + :param host: ArangoDB host URL. + :type host: str | unicode + :returns: Requests session object. + :rtype: requests.Session + """ + raise NotImplementedError + @abstractmethod def send_request(self, + session, method, url, headers=None, @@ -26,6 +40,8 @@ def send_request(self, This method must be overridden by the user. + :param session: Requests session object. + :type session: requests.Session :param method: HTTP method in lowercase (e.g. "post"). :type method: str | unicode :param url: Request URL. @@ -47,10 +63,18 @@ def send_request(self, class DefaultHTTPClient(HTTPClient): """Default HTTP client implementation.""" - def __init__(self): - self._session = requests.Session() + def create_session(self, host): + """Create and return a new session/connection. + + :param host: ArangoDB host URL. + :type host: str | unicode + :returns: requests session object + :rtype: requests.Session + """ + return requests.Session() def send_request(self, + session, method, url, params=None, @@ -59,6 +83,8 @@ def send_request(self, auth=None): """Send an HTTP request. + :param session: Requests session object. + :type session: requests.Session :param method: HTTP method in lowercase (e.g. "post"). :type method: str | unicode :param url: Request URL. @@ -74,7 +100,7 @@ def send_request(self, :returns: HTTP response. :rtype: arango.response.Response """ - raw_resp = self._session.request( + response = session.request( method=method, url=url, params=params, @@ -83,10 +109,10 @@ def send_request(self, auth=auth, ) return Response( - method=raw_resp.request.method, - url=raw_resp.url, - headers=raw_resp.headers, - status_code=raw_resp.status_code, - status_text=raw_resp.reason, - raw_body=raw_resp.text, + method=response.request.method, + url=response.url, + headers=response.headers, + status_code=response.status_code, + status_text=response.reason, + raw_body=response.text, ) diff --git a/arango/job.py b/arango/job.py index b9e468cc..def131cc 100644 --- a/arango/job.py +++ b/arango/job.py @@ -7,8 +7,7 @@ AsyncJobStatusError, AsyncJobResultError, AsyncJobClearError, - BatchJobResultError, - TransactionJobResultError, + BatchJobResultError ) from arango.request import Request @@ -238,55 +237,3 @@ def result(self): if self._status == 'pending': raise BatchJobResultError('result not available yet') return self._response_handler(self._response) - - -class TransactionJob(Job): - """Transaction API execution job. - - :param response_handler: HTTP response handler. - :type response_handler: callable - """ - - __slots__ = ['_id', '_status', '_response', '_response_handler'] - - def __init__(self, response_handler): - self._id = uuid4().hex - self._status = 'pending' - self._response = None - self._response_handler = response_handler - - def __repr__(self): - return ''.format(self._id) - - @property - def id(self): - """Return the transaction job ID. - - :return: Transaction job ID. - :rtype: str | unicode - """ - return self._id - - def status(self): - """Return the transaction job status. - - :return: Transaction job status. Possible values are "pending" (job is - waiting for transaction to be committed, or transaction failed and - job is orphaned), or "done" (transaction was committed and job is - updated with the result). - :rtype: str | unicode - """ - return self._status - - def result(self): - """Return the transaction job result. - - :return: Transaction job result. - :rtype: str | unicode | bool | int | list | dict - :raise arango.exceptions.ArangoError: If the job raised an exception. - :raise arango.exceptions.TransactionJobResultError: If job result is - not available (i.e. transaction is not committed yet or failed). - """ - if self._status == 'pending': - raise TransactionJobResultError('result not available yet') - return self._response_handler(self._response) diff --git a/arango/request.py b/arango/request.py index fe98b7c3..d7752ccf 100644 --- a/arango/request.py +++ b/arango/request.py @@ -2,10 +2,6 @@ __all__ = ['Request'] -import json - -from six import moves, string_types - class Request(object): """HTTP request. @@ -20,12 +16,14 @@ class Request(object): :type params: dict :param data: Request payload. :type data: str | unicode | bool | int | list | dict - :param command: ArangoSh command. - :type command: str | unicode :param read: Names of collections read during transaction. :type read: str | unicode | [str | unicode] - :param write: Names of collections written to during transaction. + :param write: Name(s) of collections written to during transaction with + shared access. :type write: str | unicode | [str | unicode] + :param exclusive: Name(s) of collections written to during transaction + with exclusive access. + :type exclusive: str | unicode | [str | unicode] :ivar method: HTTP method in lowercase (e.g. "post"). :vartype method: str | unicode @@ -37,12 +35,14 @@ class Request(object): :vartype params: dict :ivar data: Request payload. :vartype data: str | unicode | bool | int | list | dict - :ivar command: ArangoSh command. - :vartype command: str | unicode | None :ivar read: Names of collections read during transaction. :vartype read: str | unicode | [str | unicode] | None - :ivar write: Names of collections written to during transaction. + :ivar write: Name(s) of collections written to during transaction with + shared access. :vartype write: str | unicode | [str | unicode] | None + :ivar exclusive: Name(s) of collections written to during transaction + with exclusive access. + :vartype exclusive: str | unicode | [str | unicode] | None """ __slots__ = ( @@ -51,9 +51,9 @@ class Request(object): 'headers', 'params', 'data', - 'command', 'read', - 'write' + 'write', + 'exclusive' ) def __init__(self, @@ -62,9 +62,9 @@ def __init__(self, headers=None, params=None, data=None, - command=None, read=None, - write=None): + write=None, + exclusive=None): self.method = method self.endpoint = endpoint self.headers = headers or {} @@ -78,30 +78,9 @@ def __init__(self, for key, val in params.items(): if isinstance(val, bool): params[key] = int(val) - self.params = params - # Normalize the payload. - if data is None: - self.data = None - elif isinstance(data, string_types): - self.data = data - else: - self.data = json.dumps(data) - - # Set the transaction metadata. - self.command = command + self.params = params + self.data = data self.read = read self.write = write - - def __str__(self): - """Return the request details in string form.""" - path = self.endpoint - if self.params is not None: - path += '?' + moves.urllib.parse.urlencode(self.params) - request_strings = ['{} {} HTTP/1.1'.format(self.method, path)] - if self.headers is not None: - for key, value in sorted(self.headers.items()): - request_strings.append('{}: {}'.format(key, value)) - if self.data is not None: - request_strings.append('\r\n{}'.format(self.data)) - return '\r\n'.join(request_strings) + self.exclusive = exclusive diff --git a/arango/resolver.py b/arango/resolver.py new file mode 100644 index 00000000..363a898c --- /dev/null +++ b/arango/resolver.py @@ -0,0 +1,49 @@ +from __future__ import absolute_import, unicode_literals + +__all__ = [ + 'SingleHostResolver', + 'RandomHostResolver', + 'RoundRobinHostResolver', +] + +import random +from abc import ABCMeta, abstractmethod + + +class HostResolver(object): # pragma: no cover + """Abstract base class for host resolvers.""" + + __metaclass__ = ABCMeta + + @abstractmethod + def get_host_index(self): + raise NotImplementedError + + +class SingleHostResolver(HostResolver): + """Single host resolver.""" + + def get_host_index(self): + return 0 + + +class RandomHostResolver(HostResolver): + """Random host resolver.""" + + def __init__(self, host_count): + self._max = host_count - 1 + + def get_host_index(self): + return random.randint(0, self._max) + + +class RoundRobinHostResolver(HostResolver): + """Round-robin host resolver.""" + + def __init__(self, host_count): + self._index = -1 + self._count = host_count + + def get_host_index(self): + self._index = (self._index + 1) % self._count + return self._index diff --git a/arango/response.py b/arango/response.py index 414df9cc..b4509c07 100644 --- a/arango/response.py +++ b/arango/response.py @@ -2,8 +2,6 @@ __all__ = ['Response'] -import json - class Response(object): """HTTP response. @@ -31,10 +29,10 @@ class Response(object): :vartype status_code: int :ivar status_text: Response status text. :vartype status_text: str | unicode - :ivar body: JSON-deserialized response body. - :vartype body: str | unicode | bool | int | list | dict :ivar raw_body: Raw response body. :vartype raw_body: str | unicode + :ivar body: JSON-deserialized response body. + :vartype body: str | unicode | bool | int | list | dict :ivar error_code: Error code from ArangoDB server. :vartype error_code: int :ivar error_message: Error message from ArangoDB server. @@ -70,19 +68,8 @@ def __init__(self, self.status_text = status_text self.raw_body = raw_body - # De-serialize the response body. - try: - self.body = json.loads(raw_body) - except (ValueError, TypeError): - self.body = raw_body - - # Extract error code and message. - if isinstance(self.body, dict): - self.error_code = self.body.get('errorNum') - self.error_message = self.body.get('errorMessage') - else: - self.error_code = None - self.error_message = None - - http_ok = 200 <= status_code < 300 - self.is_success = http_ok and self.error_code is None + # Populated later + self.body = None + self.error_code = None + self.error_message = None + self.is_success = None diff --git a/arango/version.py b/arango/version.py index 26a6c390..a0f66580 100644 --- a/arango/version.py +++ b/arango/version.py @@ -1 +1 @@ -__version__ = '4.4.0' +__version__ = '5.0.0' diff --git a/docs/admin.rst b/docs/admin.rst index 4beb2479..76cd4452 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -17,9 +17,6 @@ database. # Connect to "_system" database as root user. sys_db = client.db('_system', username='root', password='passwd') - # Check the server connection by sending a test GET request. - sys_db.ping() - # Retrieve the server version. sys_db.version() diff --git a/docs/analyzer.rst b/docs/analyzer.rst new file mode 100644 index 00000000..17e70bbd --- /dev/null +++ b/docs/analyzer.rst @@ -0,0 +1,35 @@ +Analyzers +--------- + +Python-arango supports **analyzers**. For more information on analyzers, refer +to `ArangoDB manual`_. + +.. _ArangoDB manual: https://docs.arangodb.com + +**Example:** + +.. testcode:: + + from arango import ArangoClient + + # Initialize the ArangoDB client. + client = ArangoClient() + + # Connect to "test" database as root user. + db = client.db('test', username='root', password='passwd') + + # Retrieve list of analyzers. + db.analyzers() + + # Create an analyzer. + db.create_analyzer( + name='test_analyzer', + analyzer_type='identity', + properties={}, + features=[] + ) + + # Delete an analyzer. + db.delete_analyzer('test_analyzer', ignore_missing=True) + +Refer to :ref:`StandardDatabase` class for API specification. \ No newline at end of file diff --git a/docs/aql.rst b/docs/aql.rst index 2ff84f91..a7f39d2c 100644 --- a/docs/aql.rst +++ b/docs/aql.rst @@ -74,7 +74,7 @@ AQL queries are invoked from AQL API wrapper. Executing queries returns try: aql.kill('some_query_id') except AQLQueryKillError as err: - assert err.http_code == 400 + assert err.http_code == 404 assert err.error_code == 1591 assert 'cannot kill query' in err.message @@ -143,9 +143,9 @@ are not. aql.cache.properties() # Configure AQL query cache properties - aql.cache.configure(mode='demand', limit=10000) + aql.cache.configure(mode='demand', max_results=10000) # Clear results in AQL query cache. aql.cache.clear() -See :ref:`AQLQueryCache` for API specification. +See :ref:`AQLQueryCache` for API specification. \ No newline at end of file diff --git a/docs/async.rst b/docs/async.rst index f27e5547..e2be76a4 100644 --- a/docs/async.rst +++ b/docs/async.rst @@ -1,12 +1,12 @@ -Async Execution ---------------- +Asynchronous Execution +---------------------- -Python-arango supports **async execution**, where it sends requests to ArangoDB -server in fire-and-forget style (HTTP 202 returned). The server places incoming -requests in its queue and processes them in the background. The results can be -retrieved from the server later via :ref:`AsyncJob` objects. +In **asynchronous execution**, python-arango sends API requests to ArangoDB in +fire-and-forget style. The server processes the requests in the background, and +the results can be retrieved once available via :ref:`AsyncJob` objects. -**Example:** +**Example +:** .. testcode:: diff --git a/docs/batch.rst b/docs/batch.rst index 5db8229a..ec932d9a 100644 --- a/docs/batch.rst +++ b/docs/batch.rst @@ -1,9 +1,9 @@ Batch Execution --------------- -Python-arango supports **batch execution**. Requests to ArangoDB server are -placed in client-side in-memory queue, and committed together in a single HTTP -call. After the commit, results can be retrieved from :ref:`BatchJob` objects. +In **batch execution**, requests to ArangoDB server are stored in client-side +in-memory queue, and committed together in a single HTTP call. After the commit, +results can be retrieved later from :ref:`BatchJob` objects. **Example:** diff --git a/docs/cursor.rst b/docs/cursor.rst index f5c747ef..10d12292 100644 --- a/docs/cursor.rst +++ b/docs/cursor.rst @@ -116,42 +116,3 @@ instead. cursor.fetch() while not cursor.empty(): # Pop until nothing is left on the cursor. cursor.pop() - -When running queries in :doc:`transactions `, cursors are loaded -with the entire result set right away. This is regardless of the parameters -passed in when executing the query (e.g. batch_size). You must be mindful of -client-side memory capacity when executing queries that can potentially return -a large result set. - -**Example:** - -.. testcode:: - - # Initialize the ArangoDB client. - client = ArangoClient() - - # Connect to "test" database as root user. - db = client.db('test', username='root', password='passwd') - - # Get the total document count in "students" collection. - document_count = db.collection('students').count() - - # Execute an AQL query normally (without using transactions). - cursor1 = db.aql.execute('FOR doc IN students RETURN doc', batch_size=1) - - # Execute the same AQL query in a transaction. - txn_db = db.begin_transaction() - job = txn_db.aql.execute('FOR doc IN students RETURN doc', batch_size=1) - txn_db.commit() - cursor2 = job.result() - - # The first cursor acts as expected. Its current batch contains only 1 item - # and it still needs to fetch the rest of its result set from the server. - assert len(cursor1.batch()) == 1 - assert cursor1.has_more() is True - - # The second cursor is pre-loaded with the entire result set, and does not - # require further communication with ArangoDB server. Note that value of - # parameter "batch_size" was ignored. - assert len(cursor2.batch()) == document_count - assert cursor2.has_more() is False diff --git a/docs/database.rst b/docs/database.rst index 968cedf8..cc90def3 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -15,7 +15,7 @@ information. from arango import ArangoClient # Initialize the ArangoDB client. - client = ArangoClient(protocol='http', host='localhost', port=8529) + client = ArangoClient() # Connect to "_system" database as root user. # This returns an API wrapper for "_system" database. diff --git a/docs/document.rst b/docs/document.rst index 4702c9ca..62ad0886 100644 --- a/docs/document.rst +++ b/docs/document.rst @@ -56,7 +56,7 @@ document in "friends" edge collection: { '_id': 'friends/001', '_key': '001', - '_rev': '_Wm3dyle--_', + '_rev': '_Wm3d4le--_', '_from': 'students/john', '_to': 'students/jane', 'closeness': 9.5 diff --git a/docs/errors.rst b/docs/errors.rst index 69fc34b3..53b16ac1 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -63,9 +63,6 @@ message, error code and HTTP request response details. request.headers # Request headers request.params # URL parameters request.data # Request payload - request.read # Read collections (used for transactions only) - request.write # Write collections (used for transactions only) - request.command # ArangoSh command (used for transactions only) See :ref:`Response` and :ref:`Request` for reference. diff --git a/docs/foxx.rst b/docs/foxx.rst index 3c2308ea..13dc4955 100644 --- a/docs/foxx.rst +++ b/docs/foxx.rst @@ -1,15 +1,14 @@ Foxx ---- -Python-arango supports **Foxx**, a microservice framework which lets you define -custom HTTP endpoints to extend ArangoDB's REST API. For more information, refer -to `ArangoDB manual`_. +Python-arango provides support for **Foxx**, a microservice framework which +lets you define custom HTTP endpoints to extend ArangoDB's REST API. For more +information, refer to `ArangoDB manual`_. .. _ArangoDB manual: https://docs.arangodb.com **Example:** - .. testcode:: from arango import ArangoClient diff --git a/docs/http.rst b/docs/http.rst index 2ede1d7b..2bdd72ce 100644 --- a/docs/http.rst +++ b/docs/http.rst @@ -1,12 +1,19 @@ -Using Custom HTTP Clients -------------------------- +HTTP Clients +------------ -Python-arango lets you use your own HTTP clients for sending API requests to +Python-arango lets you define your own HTTP client for sending requests to ArangoDB server. The default implementation uses the requests_ library. Your HTTP client must inherit :class:`arango.http.HTTPClient` and implement its -abstract method :func:`arango.http.HTTPClient.send_request`. The method must -return valid (fully populated) instances of :class:`arango.response.Response`. +two abstract methods: + +* :func:`arango.http.HTTPClient.create_session` +* :func:`arango.http.HTTPClient.send_request` + +The **create_session** method must return a session object (called per host) +which will be stored in python-arango client. The **send_request** method must +use the session to send a HTTP request, and return a fully populated instance +of :class:`arango.response.Response`. For example, let's say your HTTP client needs: @@ -32,31 +39,34 @@ Your ``CustomHTTPClient`` class might look something like this: """My custom HTTP client with cool features.""" def __init__(self): - self._session = Session() - # Initialize your logger. self._logger = logging.getLogger('my_logger') - # Add your request headers. - self._session.headers.update({'x-my-header': 'true'}) + def create_session(self, host): + session = Session() + + # Add request header. + session.headers.update({'x-my-header': 'true'}) # Enable retries. adapter = HTTPAdapter(max_retries=5) self._session.mount('https://', adapter) + return session + def send_request(self, + session, method, url, params=None, data=None, headers=None, auth=None): - # Add your own debug statement. self._logger.debug('Sending request to {}'.format(url)) # Send a request. - response = self._session.request( + response = session.request( method=method, url=url, params=params, @@ -67,7 +77,7 @@ Your ``CustomHTTPClient`` class might look something like this: ) self._logger.debug('Got {}'.format(response.status_code)) - # Return an instance of arango.response.Response per spec. + # Return an instance of arango.response.Response. return Response( method=response.request.method, url=response.url, @@ -79,74 +89,14 @@ Your ``CustomHTTPClient`` class might look something like this: Then you would inject your client as follows: -.. testsetup:: - - import logging - - from requests.adapters import HTTPAdapter - from requests import Session - - from arango.response import Response - from arango.http import HTTPClient - - class CustomHTTPClient(HTTPClient): - """Custom HTTP client.""" - - def __init__(self): - self._session = Session() - - # Initialize logger. - self._logger = logging.getLogger('my_logger') - - # Add request headers. - self._session.headers.update({'x-my-header': 'true'}) - - # Add retries. - adapter = HTTPAdapter(max_retries=5) - self._session.mount('https://', adapter) - - def send_request(self, - method, - url, - params=None, - data=None, - headers=None, - auth=None): - # Add your own debug statement. - self._logger.debug('Sending request to {}'.format(url)) - - # Send a request without SSL verification. - response = self._session.request( - method=method, - url=url, - params=params, - data=data, - headers=headers, - auth=auth, - verify=False # No SSL verification - ) - self._logger.debug('Got {}'.format(response.status_code)) - - # You must return an instance of arango.response.Response. - return Response( - method=response.request.method, - url=response.url, - headers=response.headers, - status_code=response.status_code, - status_text=response.reason, - raw_body=response.text, - ) - -.. testcode:: +.. code-block:: python from arango import ArangoClient - # from my_module import CustomHTTPClient + from my_module import CustomHTTPClient client = ArangoClient( - protocol='http', - host='localhost', - port=8529, + hosts='http://localhost:8529', http_client=CustomHTTPClient() ) @@ -154,4 +104,4 @@ For more information on how to configure a ``requests.Session`` object, refer to `requests documentation`_. .. _requests: https://github.com/requests/requests -.. _requests documentation: http://docs.python-requests.org/en/master/user/advanced/#session-objects \ No newline at end of file +.. _requests documentation: http://docs.python-requests.org/en/master/user/advanced/#session-objects diff --git a/docs/index.rst b/docs/index.rst index d4756543..8bf6f723 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,15 +8,16 @@ Welcome to the documentation for **python-arango**, a Python driver for ArangoDB Features ======== -- Clean Pythonic interface +- Pythonic interface - Lightweight -- High ArangoDB REST API coverage +- High API coverage Compatibility ============= -- Python versions 2.7, 3.4, 3.5 and 3.6 are supported -- Python-arango 4.x supports ArangoDB 3.3+ (recommended) +- Python versions 2.7, 3.4, 3.5, 3.6 and 3.7 are supported +- Python-arango 5.x supports ArangoDB 3.5+ +- Python-arango 4.x supports ArangoDB 3.3 ~ 3.4 only - Python-arango 3.x supports ArangoDB 3.0 ~ 3.2 only - Python-arango 2.x supports ArangoDB 1.x ~ 2.x only @@ -68,9 +69,11 @@ Contents pregel foxx view + analyzer threading errors logging http + serializer contributing specs diff --git a/docs/indexes.rst b/docs/indexes.rst index d1a10c32..5705517a 100644 --- a/docs/indexes.rst +++ b/docs/indexes.rst @@ -40,9 +40,15 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to # Add a new geo-spatial index on field 'coordinates'. index = cities.add_geo_index(fields=['coordinates']) - # Add a new persistent index on fields 'currency'. + # Add a new persistent index on field 'currency'. index = cities.add_persistent_index(fields=['currency'], sparse=True) + # Add a new TTL (time-to-live) index on field 'currency'. + index = cities.add_ttl_index(fields=['ttl'], expiry_time=200) + + # Indexes may be added with a name that can be referred to in AQL queries. + index = cities.add_hash_index(fields=['country'], name='my_hash_index') + # Delete the last index from the collection. cities.delete_index(index['id']) diff --git a/docs/logging.rst b/docs/logging.rst index 79d6724a..2ffd693d 100644 --- a/docs/logging.rst +++ b/docs/logging.rst @@ -1,8 +1,8 @@ Logging ------- -In order to see full HTTP request response details, you can modify logger -settings for Requests_ library, which python-arango uses under the hood: +To see full HTTP request and response details, you can modify the logger +settings for the Requests_ library, which python-arango uses under the hood: .. _Requests: https://github.com/requests/requests @@ -25,9 +25,5 @@ settings for Requests_ library, which python-arango uses under the hood: requests_log.setLevel(logging.DEBUG) requests_log.propagate = True -.. note:: - If python-arango's default HTTP client is overridden with a custom one, - the code snippet above may not work as expected. - -Alternatively, if you want to use your own loggers, see :doc:`http` for an -example. +If python-arango's default HTTP client is overridden, the code snippet above +may not work as expected. See :doc:`http` for more information. diff --git a/docs/overview.rst b/docs/overview.rst index 3d65bd80..76ff4155 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -8,7 +8,7 @@ Here is an example showing how **python-arango** client can be used: from arango import ArangoClient # Initialize the ArangoDB client. - client = ArangoClient(protocol='http', host='localhost', port=8529) + client = ArangoClient(hosts='http://localhost:8529') # Connect to "_system" database as root user. # This returns an API wrapper for "_system" database. diff --git a/docs/pregel.rst b/docs/pregel.rst index 15da7742..45c55f4a 100644 --- a/docs/pregel.rst +++ b/docs/pregel.rst @@ -1,8 +1,8 @@ Pregel ------ -Python-arango supports **Pregel**, an ArangoDB module for distributed iterative -graph processing. For more information, refer to `ArangoDB manual`_. +Python-arango provides support for **Pregel**, ArangoDB module for distributed +iterative graph processing. For more information, refer to `ArangoDB manual`_. .. _ArangoDB manual: https://docs.arangodb.com diff --git a/docs/serializer.rst b/docs/serializer.rst new file mode 100644 index 00000000..6b67bf1e --- /dev/null +++ b/docs/serializer.rst @@ -0,0 +1,22 @@ +JSON Serialization +------------------ + +You can provide your own JSON serializer and deserializer during client +initialization. They must be callables that take a single argument. + +**Example:** + +.. testcode:: + + import json + + from arango import ArangoClient + + # Initialize the ArangoDB client with custom serializer and deserializer. + client = ArangoClient( + hosts='http://localhost:8529', + serializer=json.dumps, + deserializer=json.loads + ) + +See :ref:`ArangoClient` for API specification. \ No newline at end of file diff --git a/docs/specs.rst b/docs/specs.rst index e2499697..9a0af8e8 100644 --- a/docs/specs.rst +++ b/docs/specs.rst @@ -161,14 +161,6 @@ TransactionDatabase :inherited-members: :members: -.. _TransactionJob: - -TransactionJob -============== - -.. autoclass:: arango.job.TransactionJob - :members: - .. _VertexCollection: VertexCollection diff --git a/docs/threading.rst b/docs/threading.rst index 43c18ac0..294559d2 100644 --- a/docs/threading.rst +++ b/docs/threading.rst @@ -7,7 +7,5 @@ shared across multiple threads without locks in place: * :ref:`BatchDatabase` (see :doc:`batch`) * :ref:`BatchJob` (see :doc:`batch`) * :ref:`Cursor` (see :doc:`cursor`) -* :ref:`TransactionDatabase` (see :doc:`transaction`) -* :ref:`TransactionJob` (see :doc:`transaction`) The rest of python-arango is safe to use in multi-threaded environments. diff --git a/docs/transaction.rst b/docs/transaction.rst index debc3422..84d03cc4 100644 --- a/docs/transaction.rst +++ b/docs/transaction.rst @@ -1,146 +1,85 @@ Transactions ------------ -Python-arango supports **transactions**, where requests to ArangoDB server are -placed in client-side in-memory queue, and committed as a single, logical unit -of work (ACID compliant). After a successful commit, results can be retrieved -from :ref:`TransactionJob` objects. +In **transactions**, requests to ArangoDB server are committed as a single, +logical unit of work (ACID compliant). -**Example:** - -.. testcode:: - - from arango import ArangoClient - - # Initialize the ArangoDB client. - client = ArangoClient() - - # Connect to "test" database as root user. - db = client.db('test', username='root', password='passwd') - - # Get the API wrapper for "students" collection. - students = db.collection('students') - - # Begin a transaction via context manager. This returns an instance of - # TransactionDatabase, a database-level API wrapper tailored specifically - # for executing transactions. The transaction is automatically committed - # when exiting the context. The TransactionDatabase wrapper cannot be - # reused after commit and may be discarded after. - with db.begin_transaction() as txn_db: - - # Child wrappers are also tailored for transactions. - txn_col = txn_db.collection('students') - - # API execution context is always set to "transaction". - assert txn_db.context == 'transaction' - assert txn_col.context == 'transaction' - - # TransactionJob objects are returned instead of results. - job1 = txn_col.insert({'_key': 'Abby'}) - job2 = txn_col.insert({'_key': 'John'}) - job3 = txn_col.insert({'_key': 'Mary'}) - - # Upon exiting context, transaction is automatically committed. - assert 'Abby' in students - assert 'John' in students - assert 'Mary' in students - - # Retrieve the status of each transaction job. - for job in txn_db.queued_jobs(): - # Status is set to either "pending" (transaction is not committed yet - # and result is not available) or "done" (transaction is committed and - # result is available). - assert job.status() in {'pending', 'done'} - - # Retrieve the job results. - metadata = job1.result() - assert metadata['_id'] == 'students/Abby' +.. warning:: - metadata = job2.result() - assert metadata['_id'] == 'students/John' - - metadata = job3.result() - assert metadata['_id'] == 'students/Mary' - - # Transactions can be initiated without using a context manager. - # If return_result parameter is set to False, no jobs are returned. - txn_db = db.begin_transaction(return_result=False) - txn_db.collection('students').insert({'_key': 'Jake'}) - txn_db.collection('students').insert({'_key': 'Jill'}) - - # The commit must be called explicitly. - txn_db.commit() - assert 'Jake' in students - assert 'Jill' in students - -.. note:: - * Be mindful of client-side memory capacity when issuing a large number of - requests in a single transaction. - * :ref:`TransactionDatabase` and :ref:`TransactionJob` instances are - stateful objects, and should not be shared across multiple threads. - * :ref:`TransactionDatabase` instance cannot be reused after commit. - -See :ref:`TransactionDatabase` and :ref:`TransactionJob` for API specification. - -Error Handling -============== - -Unlike :doc:`batch ` or :doc:`async ` execution, job-specific -error handling is not possible for transactions. As soon as a job fails, the -entire transaction is halted, all previous successful jobs are rolled back, -and :class:`arango.exceptions.TransactionExecuteError` is raised. The exception -describes the first failed job, and all :ref:`TransactionJob` objects are left -at "pending" status (they may be discarded). + New transaction REST API was added to ArangoDB version 3.5. In order to use + it python-arango's own transaction API had to be overhauled in version + 5.0.0. **The changes are not backward-compatible**: context managers are no + longer offered (you must always commit the transaction youself), method + signatures are different when beginning the transaction, and results are + returned immediately instead of job objects. **Example:** .. testcode:: - from arango import ArangoClient, TransactionExecuteError + from arango import ArangoClient # Initialize the ArangoDB client. client = ArangoClient() # Connect to "test" database as root user. db = client.db('test', username='root', password='passwd') + col = db.collection('students') - # Get the API wrapper for "students" collection. - students = db.collection('students') - - # Begin a new transaction. - txn_db = db.begin_transaction() - txn_col = txn_db.collection('students') - - job1 = txn_col.insert({'_key': 'Karl'}) # Is going to be rolled back. - job2 = txn_col.insert({'_key': 'Karl'}) # Fails due to duplicate key. - job3 = txn_col.insert({'_key': 'Josh'}) # Never executed on the server. + # Begin a transaction. Read and write collections must be declared ahead of + # time. This returns an instance of TransactionDatabase, database-level + # API wrapper tailored specifically for executing transactions. + txn_db = db.begin_transaction(read=col.name, write=col.name) - try: - txn_db.commit() - except TransactionExecuteError as err: - assert err.http_code == 409 - assert err.error_code == 1210 - assert err.message.endswith('conflicting key: Karl') + # The API wrapper is specific to a single transaction with a unique ID. + txn_db.transaction_id - # All operations in the transaction are rolled back. - assert 'Karl' not in students - assert 'Josh' not in students + # Child wrappers are also tailored only for the specific transaction. + txn_aql = txn_db.aql + txn_col = txn_db.collection('students') - # All transaction jobs are left at "pending "status and may be discarded. - for job in txn_db.queued_jobs(): - assert job.status() == 'pending' + # API execution context is always set to "transaction". + assert txn_db.context == 'transaction' + assert txn_aql.context == 'transaction' + assert txn_col.context == 'transaction' + + # From python-arango version 5+, results are returned immediately instead + # of job objects on API execution. + assert '_rev' in txn_col.insert({'_key': 'Abby'}) + assert '_rev' in txn_col.insert({'_key': 'John'}) + assert '_rev' in txn_col.insert({'_key': 'Mary'}) + + # Check the transaction status. + txn_db.transaction_status() + + # Commit the transaction. + txn_db.commit_transaction() + assert 'Abby' in col + assert 'John' in col + assert 'Mary' in col + assert len(col) == 3 + + # Begin another transaction. Note that the wrappers above are specific to + # the last transaction and cannot be reused. New ones must be created. + txn_db = db.begin_transaction(read=col.name, write=col.name) + txn_col = txn_db.collection('students') + assert '_rev' in txn_col.insert({'_key': 'Kate'}) + assert '_rev' in txn_col.insert({'_key': 'Mike'}) + assert '_rev' in txn_col.insert({'_key': 'Lily'}) + assert len(txn_col) == 6 -Restrictions -============ + # Abort the transaction + txn_db.abort_transaction() + assert 'Kate' not in col + assert 'Mike' not in col + assert 'Lily' not in col + assert len(col) == 3 # transaction is aborted so txn_col cannot be used -This section covers important restrictions that you must keep in mind before -choosing to use transactions. +See :ref:`TransactionDatabase` for API specification. -:ref:`TransactionJob` results are available only *after* commit, and are not -accessible during execution. If you need to implement a logic which depends on -intermediate, in-transaction values, you can instead call the method -:func:`arango.database.Database.execute_transaction` which takes raw Javascript -command as its argument. +Alternatively, you can use +:func:`arango.database.StandardDatabase.execute_transaction` to run raw +Javascript code in a transaction. **Example:** @@ -182,145 +121,4 @@ command as its argument. assert result is True assert 'Lucy' in students assert 'Greg' in students - assert 'Dona' not in students - -Note that in above example, :func:`arango.database.Database.execute_transaction` -requires names of *read* and *write* collections as python-arango has no way of -reliably figuring out which collections are used. This is also the case when -executing AQL queries. - -**Example:** - -.. testcode:: - - from arango import ArangoClient - - # Initialize the ArangoDB client. - client = ArangoClient() - - # Connect to "test" database as root user. - db = client.db('test', username='root', password='passwd') - - # Begin a new transaction via context manager. - with db.begin_transaction() as txn_db: - job = txn_db.aql.execute( - 'INSERT {_key: "Judy", age: @age} IN students RETURN true', - bind_vars={'age': 19}, - # You must specify the "read" and "write" collections. - read_collections=[], - write_collections=['students'] - ) - cursor = job.result() - assert cursor.next() is True - assert db.collection('students').get('Judy')['age'] == 19 - -Due to limitations of ArangoDB's REST API, only the following methods are -supported in transactions: - -* :func:`arango.aql.AQL.execute` -* :func:`arango.collection.StandardCollection.get` -* :func:`arango.collection.StandardCollection.get_many` -* :func:`arango.collection.StandardCollection.insert` -* :func:`arango.collection.StandardCollection.insert_many` -* :func:`arango.collection.StandardCollection.update` -* :func:`arango.collection.StandardCollection.update_many` -* :func:`arango.collection.StandardCollection.update_match` -* :func:`arango.collection.StandardCollection.replace` -* :func:`arango.collection.StandardCollection.replace_many` -* :func:`arango.collection.StandardCollection.replace_match` -* :func:`arango.collection.StandardCollection.delete` -* :func:`arango.collection.StandardCollection.delete_many` -* :func:`arango.collection.StandardCollection.delete_match` -* :func:`arango.collection.StandardCollection.properties` -* :func:`arango.collection.StandardCollection.statistics` -* :func:`arango.collection.StandardCollection.revision` -* :func:`arango.collection.StandardCollection.checksum` -* :func:`arango.collection.StandardCollection.rotate` -* :func:`arango.collection.StandardCollection.truncate` -* :func:`arango.collection.StandardCollection.count` -* :func:`arango.collection.StandardCollection.has` -* :func:`arango.collection.StandardCollection.ids` -* :func:`arango.collection.StandardCollection.keys` -* :func:`arango.collection.StandardCollection.all` -* :func:`arango.collection.StandardCollection.find` -* :func:`arango.collection.StandardCollection.find_near` -* :func:`arango.collection.StandardCollection.find_in_range` -* :func:`arango.collection.StandardCollection.find_in_radius` -* :func:`arango.collection.StandardCollection.find_in_box` -* :func:`arango.collection.StandardCollection.find_by_text` -* :func:`arango.collection.StandardCollection.get_many` -* :func:`arango.collection.StandardCollection.random` -* :func:`arango.collection.StandardCollection.indexes` -* :func:`arango.collection.VertexCollection.get` -* :func:`arango.collection.VertexCollection.insert` -* :func:`arango.collection.VertexCollection.update` -* :func:`arango.collection.VertexCollection.replace` -* :func:`arango.collection.VertexCollection.delete` -* :func:`arango.collection.EdgeCollection.get` -* :func:`arango.collection.EdgeCollection.insert` -* :func:`arango.collection.EdgeCollection.update` -* :func:`arango.collection.EdgeCollection.replace` -* :func:`arango.collection.EdgeCollection.delete` - -If an unsupported method is called, :class:`arango.exceptions.TransactionStateError` -is raised. - -**Example:** - -.. testcode:: - - from arango import ArangoClient, TransactionStateError - - # Initialize the ArangoDB client. - client = ArangoClient() - - # Connect to "test" database as root user. - db = client.db('test', username='root', password='passwd') - - # Begin a new transaction. - txn_db = db.begin_transaction() - - # API method "databases()" is not supported and an exception is raised. - try: - txn_db.databases() - except TransactionStateError as err: - assert err.source == 'client' - assert err.message == 'action not allowed in transaction' - -When running queries in transactions, the :doc:`cursors ` are loaded -with the entire result set right away. This is regardless of the parameters -passed in when executing the query (e.g batch_size). You must be mindful of -client-side memory capacity when executing queries that can potentially return -a large result set. - -**Example:** - -.. testcode:: - - # Initialize the ArangoDB client. - client = ArangoClient() - - # Connect to "test" database as root user. - db = client.db('test', username='root', password='passwd') - - # Get the total document count in "students" collection. - document_count = db.collection('students').count() - - # Execute an AQL query normally (without using transactions). - cursor1 = db.aql.execute('FOR doc IN students RETURN doc', batch_size=1) - - # Execute the same AQL query in a transaction. - with db.begin_transaction() as txn_db: - job = txn_db.aql.execute('FOR doc IN students RETURN doc', batch_size=1) - cursor2 = job.result() - - # The first cursor acts as expected. Its current batch contains only 1 item - # and it still needs to fetch the rest of its result set from the server. - assert len(cursor1.batch()) == 1 - assert cursor1.has_more() is True - - # The second cursor is pre-loaded with the entire result set, and does not - # require further communication with ArangoDB server. Note that value of - # parameter "batch_size" was ignored. - assert len(cursor2.batch()) == document_count - assert cursor2.has_more() is False + assert 'Dona' not in students \ No newline at end of file diff --git a/docs/view.rst b/docs/view.rst index bfe181d3..f78f0ca0 100644 --- a/docs/view.rst +++ b/docs/view.rst @@ -1,5 +1,5 @@ -Views ------- +Views and ArangoSearch +---------------------- Python-arango supports **view** management. For more information on view properties, refer to `ArangoDB manual`_. @@ -55,4 +55,42 @@ properties, refer to `ArangoDB manual`_. # Delete a view. db.delete_view('bar') -Refer to :ref:`StandardDatabase` class for API specification. + +Python-arango also supports **ArangoSearch** views. + +**Example:** + +.. testcode:: + + from arango import ArangoClient + + # Initialize the ArangoDB client. + client = ArangoClient() + + # Connect to "test" database as root user. + db = client.db('test', username='root', password='passwd') + + # Create an ArangoSearch view. + db.create_arangosearch_view( + name='arangosearch_view', + properties={'cleanupIntervalStep': 0} + ) + + # Partially update an ArangoSearch view. + db.update_arangosearch_view( + name='arangosearch_view', + properties={'cleanupIntervalStep': 1000} + ) + + # Replace an ArangoSearch view. + db.replace_arangosearch_view( + name='arangosearch_view', + properties={'cleanupIntervalStep': 2000} + ) + + # ArangoSearch views can be retrieved or deleted using regular view API + db.view('arangosearch_view') + db.delete_view('arangosearch_view') + + +Refer to :ref:`StandardDatabase` class for API specification. \ No newline at end of file diff --git a/docs/wal.rst b/docs/wal.rst index 9d0568dd..37b01392 100644 --- a/docs/wal.rst +++ b/docs/wal.rst @@ -1,5 +1,5 @@ -Write-Ahead Log ---------------- +Write-Ahead Log (WAL) +--------------------- **Write-Ahead Log (WAL)** is a set of append-only files recording all writes on ArangoDB server. It is typically used to perform data recovery after a crash @@ -8,7 +8,7 @@ WAL operations can only be performed by admin users via ``_system`` database. **Example:** -.. code-block::python +.. code-block:: python from arango import ArangoClient diff --git a/tests/conftest.py b/tests/conftest.py index 5e50bfe9..f3cdc4db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ generate_string, generate_username, generate_graph_name, + empty_collection ) from tests.executors import ( TestAsyncExecutor, @@ -25,20 +26,21 @@ def pytest_addoption(parser): parser.addoption('--port', action='store', default='8529') parser.addoption('--passwd', action='store', default='passwd') parser.addoption("--complete", action="store_true") + parser.addoption("--cluster", action="store_true") # noinspection PyShadowingNames def pytest_configure(config): - client = ArangoClient( - host=config.getoption('host'), - port=config.getoption('port') + url = 'http://{}:{}'.format( + config.getoption('host'), + config.getoption('port') ) + client = ArangoClient(hosts=[url, url, url]) sys_db = client.db( name='_system', username='root', password=config.getoption('passwd') ) - # Create a user and non-system database for testing. username = generate_username() password = generate_string() @@ -52,11 +54,11 @@ def pytest_configure(config): 'password': password, }] ) - sys_db.update_permission( - username=username, - permission='rw', - database=tst_db_name - ) + # sys_db.update_permission( + # username=username, + # permission='rw', + # database=tst_db_name + # ) tst_db = client.db(tst_db_name, username, password) bad_db = client.db(bad_db_name, username, password) @@ -68,8 +70,8 @@ def pytest_configure(config): geo_index = tst_col.add_geo_index(['loc']) # Create a legacy edge collection for testing. - lecol_name = generate_col_name() - tst_db.create_collection(lecol_name, edge=True) + icol_name = generate_col_name() + tst_db.create_collection(icol_name, edge=True) # Create test vertex & edge collections and graph. graph_name = generate_graph_name() @@ -85,6 +87,7 @@ def pytest_configure(config): to_vertex_collections=[tvcol_name] ) + # noinspection PyProtectedMember global_data.update({ 'client': client, 'username': username, @@ -94,11 +97,12 @@ def pytest_configure(config): 'bad_db': bad_db, 'geo_index': geo_index, 'col_name': col_name, - 'lecol_name': lecol_name, + 'icol_name': icol_name, 'graph_name': graph_name, 'ecol_name': ecol_name, 'fvcol_name': fvcol_name, 'tvcol_name': tvcol_name, + 'cluster': False }) @@ -141,6 +145,8 @@ def pytest_generate_tests(metafunc): tst_dbs = [tst_db] bad_dbs = [bad_db] + global_data['cluster'] = metafunc.config.getoption('cluster') + if metafunc.config.getoption('complete'): tst = metafunc.module.__name__.split('.test_', 1)[-1] tst_conn = tst_db._conn @@ -150,14 +156,20 @@ def pytest_generate_tests(metafunc): # Add test transaction databases tst_txn_db = StandardDatabase(tst_conn) tst_txn_db._executor = TestTransactionExecutor(tst_conn) - tst_txn_db._is_transaction = True tst_dbs.append(tst_txn_db) bad_txn_db = StandardDatabase(bad_conn) bad_txn_db._executor = TestTransactionExecutor(bad_conn) bad_dbs.append(bad_txn_db) - if tst not in {'async', 'batch', 'transaction', 'client', 'exception'}: + if tst not in { + 'async', + 'batch', + 'transaction', + 'client', + 'exception', + 'view' + }: # Add test async databases tst_async_db = StandardDatabase(tst_conn) tst_async_db._executor = TestAsyncExecutor(tst_conn) @@ -206,10 +218,15 @@ def password(): return global_data['password'] +@pytest.fixture(autouse=False) +def conn(db): + return getattr(db, '_conn') + + @pytest.fixture(autouse=False) def col(db): collection = db.collection(global_data['col_name']) - collection.truncate() + empty_collection(collection) return collection @@ -224,9 +241,9 @@ def geo(): @pytest.fixture(autouse=False) -def lecol(db): - collection = db.collection(global_data['lecol_name']) - collection.truncate() +def icol(db): + collection = db.collection(global_data['icol_name']) + empty_collection(collection) return collection @@ -244,7 +261,7 @@ def bad_graph(bad_db): @pytest.fixture(autouse=False) def fvcol(graph): collection = graph.vertex_collection(global_data['fvcol_name']) - collection.truncate() + empty_collection(collection) return collection @@ -252,7 +269,7 @@ def fvcol(graph): @pytest.fixture(autouse=False) def tvcol(graph): collection = graph.vertex_collection(global_data['tvcol_name']) - collection.truncate() + empty_collection(collection) return collection @@ -266,7 +283,7 @@ def bad_fvcol(bad_graph): @pytest.fixture(autouse=False) def ecol(graph): collection = graph.edge_collection(global_data['ecol_name']) - collection.truncate() + empty_collection(collection) return collection @@ -316,3 +333,8 @@ def edocs(): {'_key': '3', '_from': '{}/6'.format(fv), '_to': '{}/2'.format(tv)}, {'_key': '4', '_from': '{}/8'.format(fv), '_to': '{}/7'.format(tv)}, ] + + +@pytest.fixture(autouse=False) +def cluster(): + return global_data['cluster'] diff --git a/tests/executors.py b/tests/executors.py index 4831e1de..9355a47c 100644 --- a/tests/executors.py +++ b/tests/executors.py @@ -3,9 +3,9 @@ from arango.executor import ( AsyncExecutor, BatchExecutor, - TransactionExecutor + TransactionExecutor, ) -from arango.job import BatchJob, TransactionJob +from arango.job import BatchJob class TestAsyncExecutor(AsyncExecutor): @@ -43,25 +43,24 @@ def execute(self, request, response_handler): class TestTransactionExecutor(TransactionExecutor): + # noinspection PyMissingConstructor def __init__(self, connection): - super(TestTransactionExecutor, self).__init__( - connection=connection, - timeout=0, - sync=True, - return_result=True, - read=None, - write=None - ) + self._conn = connection def execute(self, request, response_handler): - if request.command is None: - response = self._conn.send_request(request) - return response_handler(response) - - self._committed = False - self._queue.clear() + if request.read is request.write is request.exclusive is None: + resp = self._conn.send_request(request) + return response_handler(resp) - job = TransactionJob(response_handler) - self._queue[job.id] = (request, job) + super(TestTransactionExecutor, self).__init__( + connection=self._conn, + sync=True, + allow_implicit=False, + lock_timeout=0, + read=request.read, + write=request.write, + exclusive=request.exclusive + ) + result = TransactionExecutor.execute(self, request, response_handler) self.commit() - return job.result() + return result diff --git a/tests/helpers.py b/tests/helpers.py index 62c17c13..0b1574e2 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +import os from collections import deque from uuid import uuid4 @@ -9,7 +10,7 @@ from arango.exceptions import ( AsyncExecuteError, BatchExecuteError, - TransactionExecuteError + TransactionInitError ) @@ -85,6 +86,15 @@ def generate_view_name(): return 'test_view_{}'.format(uuid4().hex) +def generate_analyzer_name(): + """Generate and return a random analyzer name. + + :return: Random analyzer name. + :rtype: str | unicode + """ + return 'test_analyzer_{}'.format(uuid4().hex) + + def generate_string(): """Generate and return a random unique string. @@ -122,6 +132,16 @@ def clean_doc(obj): } +def empty_collection(collection): + """Empty all the documents in the collection. + + :param collection: Collection name + :type collection: arango.collection.StandardCollection + """ + for doc_id in collection.ids(): + collection.delete(doc_id) + + def extract(key, items): """Return the sorted values from dicts using the given key. @@ -135,16 +155,17 @@ def extract(key, items): return sorted(item[key] for item in items) -def assert_raises(*exception): +def assert_raises(*exc): """Assert that the given exception is raised. - :param exception: Expected exception(s). - :type: Exception + :param exc: Expected exception(s). + :type: exc """ + # noinspection PyTypeChecker return pytest.raises( - exception + ( - AsyncExecuteError, - BatchExecuteError, - TransactionExecuteError - ) + exc + (AsyncExecuteError, BatchExecuteError, TransactionInitError) ) + + +def file_exists(filename): + return os.path.exists(filename) diff --git a/tests/test_analyzer.py b/tests/test_analyzer.py new file mode 100644 index 00000000..834ba8b9 --- /dev/null +++ b/tests/test_analyzer.py @@ -0,0 +1,60 @@ +from __future__ import absolute_import, unicode_literals + +from arango.exceptions import ( + AnalyzerCreateError, + AnalyzerDeleteError, + AnalyzerGetError, + AnalyzerListError +) +from tests.helpers import assert_raises, generate_analyzer_name + + +def test_analyzer_management(db, bad_db, cluster): + analyzer_name = generate_analyzer_name() + full_analyzer_name = db.name + '::' + analyzer_name + bad_analyzer_name = generate_analyzer_name() + + # Test create analyzer + result = db.create_analyzer(analyzer_name, 'identity', {}) + assert result['name'] == full_analyzer_name + assert result['type'] == 'identity' + assert result['properties'] == {} + assert result['features'] == [] + + # Test create duplicate with bad database + with assert_raises(AnalyzerCreateError) as err: + bad_db.create_analyzer(analyzer_name, 'identity', {}, []) + assert err.value.error_code in {11, 1228} + + # Test get analyzer + result = db.analyzer(analyzer_name) + assert result['name'] == full_analyzer_name + assert result['type'] == 'identity' + assert result['properties'] == {} + assert result['features'] == [] + + # Test get missing analyzer + with assert_raises(AnalyzerGetError) as err: + db.analyzer(bad_analyzer_name) + assert err.value.error_code in {1202} + + # Test list analyzers + result = db.analyzers() + assert full_analyzer_name in [a['name'] for a in result] + + # Test list analyzers with bad database + with assert_raises(AnalyzerListError) as err: + bad_db.analyzers() + assert err.value.error_code in {11, 1228} + + # Test delete analyzer + assert db.delete_analyzer(analyzer_name, force=True) is True + assert full_analyzer_name not in [a['name'] for a in db.analyzers()] + + # Test delete missing analyzer + with assert_raises(AnalyzerDeleteError) as err: + db.delete_analyzer(analyzer_name) + assert err.value.error_code in {1202} + + # Test delete missing analyzer with ignore_missing set to True + assert db.delete_analyzer(analyzer_name, ignore_missing=True) is False diff --git a/tests/test_aql.py b/tests/test_aql.py index a775ddac..e8006f3c 100644 --- a/tests/test_aql.py +++ b/tests/test_aql.py @@ -86,7 +86,7 @@ def test_aql_query_management(db, bad_db, col, docs): RETURN NEW '''.format(col=col.name), count=True, - batch_size=1, + # batch_size=1, ttl=10, bind_vars={'val': 42}, full_count=True, @@ -106,30 +106,17 @@ def test_aql_query_management(db, bad_db, col, docs): stream=False, skip_inaccessible_cols=True ) - if db.context == 'transaction': - assert cursor.id is None - assert cursor.type == 'cursor' - assert cursor.batch() is not None - assert cursor.has_more() is False - assert cursor.count() == len(col) - assert cursor.cached() is None - assert cursor.statistics() is None - assert cursor.profile() is None - assert cursor.warnings() is None - assert extract('_key', cursor) == extract('_key', docs) - assert cursor.close() is None - else: - assert cursor.id is not None - assert cursor.type == 'cursor' - assert cursor.batch() is not None - assert cursor.has_more() is True - assert cursor.count() == len(col) - assert cursor.cached() is False - assert cursor.statistics() is not None - assert cursor.profile() is not None - assert cursor.warnings() == [] - assert extract('_key', cursor) == extract('_key', docs) - assert cursor.close(ignore_missing=True) is False + assert cursor.id is None + assert cursor.type == 'cursor' + assert cursor.batch() is not None + assert cursor.has_more() is False + assert cursor.count() == len(col) + assert cursor.cached() is False + assert cursor.statistics() is not None + assert cursor.profile() is not None + assert cursor.warnings() == [] + assert extract('_key', cursor) == extract('_key', docs) + assert cursor.close(ignore_missing=True) is None # Test get tracking properties with bad database with assert_raises(AQLQueryTrackingGetError) as err: @@ -310,20 +297,35 @@ def test_aql_cache_management(db, bad_db): # Test get AQL cache properties properties = db.aql.cache.properties() assert 'mode' in properties - assert 'limit' in properties + assert 'max_results' in properties + assert 'max_results_size' in properties + assert 'max_entry_size' in properties + assert 'include_system' in properties # Test get AQL cache properties with bad database with assert_raises(AQLCachePropertiesError): bad_db.aql.cache.properties() # Test get AQL cache configure properties - properties = db.aql.cache.configure(mode='on', limit=100) + properties = db.aql.cache.configure( + mode='on', + max_results=100, + max_results_size=10000, + max_entry_size=10000, + include_system=True + ) assert properties['mode'] == 'on' - assert properties['limit'] == 100 + assert properties['max_results'] == 100 + assert properties['max_results_size'] == 10000 + assert properties['max_entry_size'] == 10000 + assert properties['include_system'] is True properties = db.aql.cache.properties() assert properties['mode'] == 'on' - assert properties['limit'] == 100 + assert properties['max_results'] == 100 + assert properties['max_results_size'] == 10000 + assert properties['max_entry_size'] == 10000 + assert properties['include_system'] is True # Test get AQL cache configure properties with bad database with assert_raises(AQLCacheConfigureError): diff --git a/tests/test_async.py b/tests/test_async.py index d7cf5e36..a6b4dd2a 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -246,7 +246,7 @@ def test_async_list_jobs(db, col, docs): assert job3.id in job_ids # Test list async jobs that are pending - job4 = async_db.aql.execute('RETURN SLEEP(0.1)') + job4 = async_db.aql.execute('RETURN SLEEP(0.3)') assert db.async_jobs(status='pending') == [job4.id] wait_on_job(job4) # Make sure the job is done diff --git a/tests/test_batch.py b/tests/test_batch.py index 2d6758f7..e8e19f0b 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals +import json + import mock import pytest from six import string_types @@ -162,6 +164,8 @@ def test_batch_bad_state(db, col, docs): mock_send_request.return_value = mock_resp mock_connection = mock.MagicMock() mock_connection.send_request = mock_send_request + mock_connection.serialize = json.dumps + mock_connection.deserialize = json.loads batch_db._executor._conn = mock_connection # Test commit with invalid batch state diff --git a/tests/test_client.py b/tests/test_client.py index 8e5719c3..f5773381 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,12 +1,20 @@ from __future__ import absolute_import, unicode_literals +import json + import pytest from arango.client import ArangoClient from arango.database import StandardDatabase from arango.exceptions import ServerConnectionError from arango.http import DefaultHTTPClient +from arango.resolver import ( + SingleHostResolver, + RandomHostResolver, + RoundRobinHostResolver +) from arango.version import __version__ + from tests.helpers import ( generate_db_name, generate_username, @@ -15,27 +23,48 @@ def test_client_attributes(): - session = DefaultHTTPClient() + http_client = DefaultHTTPClient() + client = ArangoClient( - protocol='http', - host='127.0.0.1', - port=8529, - http_client=session + hosts='http://127.0.0.1:8529', + http_client=http_client ) assert client.version == __version__ - assert client.protocol == 'http' - assert client.host == '127.0.0.1' - assert client.port == 8529 - assert client.base_url == 'http://127.0.0.1:8529' + assert client.hosts == ['http://127.0.0.1:8529'] + assert repr(client) == '' + assert isinstance(client._host_resolver, SingleHostResolver) + client_repr = '' + client_hosts = ['http://127.0.0.1:8529', 'http://localhost:8529'] + + client = ArangoClient( + hosts='http://127.0.0.1:8529,http://localhost' + ':8529', + http_client=http_client, + serializer=json.dumps, + deserializer=json.loads, + ) + assert client.version == __version__ + assert client.hosts == client_hosts + assert repr(client) == client_repr + assert isinstance(client._host_resolver, RoundRobinHostResolver) -def test_client_good_connection(db, username, password): client = ArangoClient( - protocol='http', - host='127.0.0.1', - port=8529, + hosts=client_hosts, + host_resolver='random', + http_client=http_client, + serializer=json.dumps, + deserializer=json.loads, ) + assert client.version == __version__ + assert client.hosts == client_hosts + assert repr(client) == client_repr + assert isinstance(client._host_resolver, RandomHostResolver) + + +def test_client_good_connection(db, username, password): + client = ArangoClient(hosts='http://127.0.0.1:8529') # Test connection with verify flag on and off for verify in (True, False): @@ -46,23 +75,24 @@ def test_client_good_connection(db, username, password): assert db.context == 'default' -def test_client_bad_connection(db, username, password): - client = ArangoClient(protocol='http', host='127.0.0.1', port=8529) +def test_client_bad_connection(db, username, password, cluster): + client = ArangoClient(hosts='http://127.0.0.1:8529') bad_db_name = generate_db_name() bad_username = generate_username() bad_password = generate_string() - # Test connection with bad username password - with pytest.raises(ServerConnectionError): - client.db(db.name, bad_username, bad_password, verify=True) + if not cluster: + # Test connection with bad username password + with pytest.raises(ServerConnectionError): + client.db(db.name, bad_username, bad_password, verify=True) # Test connection with missing database with pytest.raises(ServerConnectionError): client.db(bad_db_name, bad_username, bad_password, verify=True) # Test connection with invalid host URL - client._url = 'http://127.0.0.1:8500' + client = ArangoClient(hosts='http://127.0.0.1:8500') with pytest.raises(ServerConnectionError) as err: client.db(db.name, username, password, verify=True) assert 'bad connection' in str(err.value) @@ -78,6 +108,7 @@ def __init__(self): self.counter = 0 def send_request(self, + session, method, url, headers=None, @@ -86,14 +117,12 @@ def send_request(self, auth=None): self.counter += 1 return super(MyHTTPClient, self).send_request( - method, url, headers, params, data, auth + session, method, url, headers, params, data, auth ) http_client = MyHTTPClient() client = ArangoClient( - protocol='http', - host='127.0.0.1', - port=8529, + hosts='http://127.0.0.1:8529', http_client=http_client ) # Set verify to True to send a test API call on initialization. diff --git a/tests/test_collection.py b/tests/test_collection.py index f1fdeed5..d97d72f6 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -17,6 +17,7 @@ CollectionCreateError, CollectionListError, CollectionDeleteError, + CollectionRecalculateCountError ) from tests.helpers import assert_raises, extract, generate_col_name @@ -29,7 +30,7 @@ def test_collection_attributes(db, col, username): assert repr(col) == ''.format(col.name) -def test_collection_misc_methods(col, bad_col): +def test_collection_misc_methods(col, bad_col, cluster): # Test get properties properties = col.properties() assert properties['name'] == col.name @@ -100,23 +101,26 @@ def test_collection_misc_methods(col, bad_col): bad_col.rotate() assert err.value.error_code in {11, 1228} - # Test checksum with empty collection - assert int(col.checksum(with_rev=True, with_data=False)) == 0 - assert int(col.checksum(with_rev=True, with_data=True)) == 0 - assert int(col.checksum(with_rev=False, with_data=False)) == 0 - assert int(col.checksum(with_rev=False, with_data=True)) == 0 - - # Test checksum with non-empty collection - col.insert({}) - assert int(col.checksum(with_rev=True, with_data=False)) > 0 - assert int(col.checksum(with_rev=True, with_data=True)) > 0 - assert int(col.checksum(with_rev=False, with_data=False)) > 0 - assert int(col.checksum(with_rev=False, with_data=True)) > 0 - - # Test checksum with bad collection - with assert_raises(CollectionChecksumError) as err: - bad_col.checksum() - assert err.value.error_code in {11, 1228} + if cluster: + col.insert({}) + else: + # Test checksum with empty collection + assert int(col.checksum(with_rev=True, with_data=False)) == 0 + assert int(col.checksum(with_rev=True, with_data=True)) == 0 + assert int(col.checksum(with_rev=False, with_data=False)) == 0 + assert int(col.checksum(with_rev=False, with_data=True)) == 0 + + # Test checksum with non-empty collection + col.insert({}) + assert int(col.checksum(with_rev=True, with_data=False)) > 0 + assert int(col.checksum(with_rev=True, with_data=True)) > 0 + assert int(col.checksum(with_rev=False, with_data=False)) > 0 + assert int(col.checksum(with_rev=False, with_data=True)) > 0 + + # Test checksum with bad collection + with assert_raises(CollectionChecksumError) as err: + bad_col.checksum() + assert err.value.error_code in {11, 1228} # Test preconditions assert len(col) == 1 @@ -125,13 +129,21 @@ def test_collection_misc_methods(col, bad_col): assert col.truncate() is True assert len(col) == 0 - # Test checksum with bad collection + # Test truncate with bad collection with assert_raises(CollectionTruncateError) as err: bad_col.truncate() assert err.value.error_code in {11, 1228} + # Test recalculate count + assert col.recalculate_count() is True + + # Test recalculate count with bad collection + with assert_raises(CollectionRecalculateCountError) as err: + bad_col.recalculate_count() + assert err.value.error_code in {11, 1228} -def test_collection_management(db, bad_db): + +def test_collection_management(db, bad_db, cluster): # Test create collection col_name = generate_col_name() assert db.has_collection(col_name) is False @@ -143,7 +155,7 @@ def test_collection_management(db, bad_db): journal_size=7774208, system=False, volatile=False, - key_generator='autoincrement', + key_generator='traditional', user_keys=False, key_increment=9, key_offset=100, @@ -154,7 +166,9 @@ def test_collection_management(db, bad_db): replication_factor=1, shard_like='', sync_replication=False, - enforce_replication_factor=False + enforce_replication_factor=False, + sharding_strategy='community-compat', + smart_join_attribute='test' ) assert db.has_collection(col_name) is True @@ -164,10 +178,8 @@ def test_collection_management(db, bad_db): assert properties['name'] == col_name assert properties['sync'] is True assert properties['system'] is False - assert properties['key_generator'] == 'autoincrement' + assert properties['key_generator'] == 'traditional' assert properties['user_keys'] is False - assert properties['key_increment'] == 9 - assert properties['key_offset'] == 100 # Test create duplicate collection with assert_raises(CollectionCreateError) as err: @@ -205,19 +217,20 @@ def test_collection_management(db, bad_db): assert err.value.error_code == 1203 assert db.delete_collection(col_name, ignore_missing=True) is False - # Test rename collection - new_name = generate_col_name() - col = db.create_collection(new_name) - assert col.rename(new_name) is True - assert col.name == new_name - assert repr(col) == ''.format(new_name) - - # Try again (the operation should be idempotent) - assert col.rename(new_name) is True - assert col.name == new_name - assert repr(col) == ''.format(new_name) - - # Test rename with bad collection - with assert_raises(CollectionRenameError) as err: - bad_db.collection(new_name).rename(new_name) - assert err.value.error_code in {11, 1228} + if not cluster: + # Test rename collection + new_name = generate_col_name() + col = db.create_collection(new_name) + assert col.rename(new_name) is True + assert col.name == new_name + assert repr(col) == ''.format(new_name) + + # Try again (the operation should be idempotent) + assert col.rename(new_name) is True + assert col.name == new_name + assert repr(col) == ''.format(new_name) + + # Test rename with bad collection + with assert_raises(CollectionRenameError) as err: + bad_db.collection(new_name).rename(new_name) + assert err.value.error_code in {11, 1228} diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 29477cd6..61adeda8 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -38,10 +38,10 @@ def test_cursor_from_execute_query(db, col, docs): assert statistics['modified'] == 0 assert statistics['filtered'] == 0 assert statistics['ignored'] == 0 - assert statistics['scanned_full'] == 6 - assert statistics['scanned_index'] == 0 assert statistics['execution_time'] > 0 - assert statistics['http_requests'] == 0 + assert 'http_requests' in statistics + assert 'scanned_full' in statistics + assert 'scanned_index' in statistics assert cursor.warnings() == [] profile = cursor.profile() @@ -119,10 +119,10 @@ def test_cursor_write_query(db, col, docs): assert statistics['modified'] == 2 assert statistics['filtered'] == 0 assert statistics['ignored'] == 0 - assert statistics['scanned_full'] == 0 - assert statistics['scanned_index'] == 2 assert statistics['execution_time'] > 0 - assert statistics['http_requests'] == 0 + assert 'http_requests' in statistics + assert 'scanned_full' in statistics + assert 'scanned_index' in statistics assert cursor.warnings() == [] profile = cursor.profile() diff --git a/tests/test_database.py b/tests/test_database.py index 88f7b239..745ed075 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -35,7 +35,7 @@ def test_database_attributes(db, username): assert repr(db) == ''.format(db.name) -def test_database_misc_methods(db, bad_db): +def test_database_misc_methods(db, bad_db, cluster): # Test get properties properties = db.properties() assert 'id' in properties @@ -198,15 +198,14 @@ def test_database_misc_methods(db, bad_db): with assert_raises(ServerLogLevelSetError): bad_db.set_log_levels(**new_levels) - # Test get server endpoints - with assert_raises(ServerEndpointsError) as err: - db.endpoints() - assert err.value.error_code in [11] + if cluster: + # Test get server endpoints + assert len(db.endpoints()) > 0 - # Test get server endpoints with bad database - with assert_raises(ServerEndpointsError) as err: - bad_db.endpoints() - assert err.value.error_code in {11, 1228} + # Test get server endpoints with bad database + with assert_raises(ServerEndpointsError) as err: + bad_db.endpoints() + assert err.value.error_code in {11, 1228} # Test get storage engine engine = db.engine() diff --git a/tests/test_document.py b/tests/test_document.py index 755025a3..effba66d 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -4,6 +4,8 @@ from six import string_types from arango.exceptions import ( + CursorNextError, + CursorCloseError, DocumentCountError, DocumentDeleteError, DocumentGetError, @@ -14,14 +16,15 @@ DocumentUpdateError, DocumentKeysError, DocumentIDsError, - DocumentParseError, + DocumentParseError ) from tests.helpers import ( assert_raises, clean_doc, extract, generate_doc_key, - generate_col_name + generate_col_name, + empty_collection ) @@ -30,7 +33,7 @@ def test_document_insert(col, docs): result = col.insert({}) assert result['_key'] in col assert len(col) == 1 - col.truncate() + empty_collection(col) # Test insert document with ID doc_id = col.name + '/' + 'foo' @@ -38,7 +41,7 @@ def test_document_insert(col, docs): assert 'foo' in col assert doc_id in col assert len(col) == 1 - col.truncate() + empty_collection(col) with assert_raises(DocumentParseError) as err: col.insert({'_id': generate_col_name() + '/' + 'foo'}) @@ -52,7 +55,7 @@ def test_document_insert(col, docs): assert isinstance(result['_rev'], string_types) assert col[doc['_key']]['val'] == doc['val'] assert len(col) == len(docs) - col.truncate() + empty_collection(col) # Test insert with sync set to True doc = docs[0] @@ -121,7 +124,7 @@ def test_document_insert_many(col, bad_col, docs): assert isinstance(result['_rev'], string_types) assert col[doc['_key']]['val'] == doc['val'] assert len(col) == len(docs) - col.truncate() + empty_collection(col) # Test insert_many with document IDs docs_with_id = [{'_id': col.name + '/' + doc['_key']} for doc in docs] @@ -131,7 +134,7 @@ def test_document_insert_many(col, bad_col, docs): assert result['_key'] == doc['_key'] assert isinstance(result['_rev'], string_types) assert len(col) == len(docs) - col.truncate() + empty_collection(col) # Test insert_many with sync set to True results = col.insert_many(docs, sync=True) @@ -141,7 +144,7 @@ def test_document_insert_many(col, bad_col, docs): assert isinstance(result['_rev'], string_types) assert col[doc['_key']]['_key'] == doc['_key'] assert col[doc['_key']]['val'] == doc['val'] - col.truncate() + empty_collection(col) # Test insert_many with sync set to False results = col.insert_many(docs, sync=False) @@ -151,7 +154,7 @@ def test_document_insert_many(col, bad_col, docs): assert isinstance(result['_rev'], string_types) assert col[doc['_key']]['_key'] == doc['_key'] assert col[doc['_key']]['val'] == doc['val'] - col.truncate() + empty_collection(col) # Test insert_many with return_new set to True results = col.insert_many(docs, return_new=True) @@ -165,7 +168,7 @@ def test_document_insert_many(col, bad_col, docs): assert result['new']['val'] == doc['val'] assert col[doc['_key']]['_key'] == doc['_key'] assert col[doc['_key']]['val'] == doc['val'] - col.truncate() + empty_collection(col) # Test insert_many with return_new set to False results = col.insert_many(docs, return_new=False) @@ -176,7 +179,7 @@ def test_document_insert_many(col, bad_col, docs): assert 'new' not in result assert col[doc['_key']]['_key'] == doc['_key'] assert col[doc['_key']]['val'] == doc['val'] - col.truncate() + empty_collection(col) # Test insert_many with silent set to True assert col.insert_many(docs, silent=True) is True @@ -959,7 +962,7 @@ def test_document_delete_many(col, bad_col, docs): assert len(col) == 6 # Test delete_many (documents) with missing documents - col.truncate() + empty_collection(col) results = col.delete_many([ {'_key': generate_doc_key()}, {'_key': generate_doc_key()}, @@ -1065,7 +1068,7 @@ def test_document_find(col, bad_col, docs): assert doc['_key'] in col # Test find in empty collection - col.truncate() + empty_collection(col) assert list(col.find({})) == [] assert list(col.find({'val': 1})) == [] assert list(col.find({'val': 2})) == [] @@ -1078,7 +1081,6 @@ def test_document_find(col, bad_col, docs): assert err.value.error_code in {11, 1228} -@pytest.mark.skip(reason='broken in ArangoDB 3.4') def test_document_find_near(col, bad_col, docs): col.import_bulk(docs) @@ -1112,7 +1114,7 @@ def test_document_find_near(col, bad_col, docs): bad_col.find_near(latitude=1, longitude=1, limit=1) # Test find_near in an empty collection - col.truncate() + empty_collection(col) result = col.find_near(latitude=1, longitude=1, limit=1) assert list(result) == [] result = col.find_near(latitude=5, longitude=5, limit=4) @@ -1188,10 +1190,7 @@ def test_document_find_in_radius(col, bad_col): distance_field='dist' )) assert len(result) == 1 - if col.context == 'transaction': - assert clean_doc(result[0]) == {'_key': '1', 'loc': [1, 1]} - else: - assert clean_doc(result[0]) == {'_key': '1', 'loc': [1, 1], 'dist': 0} + assert clean_doc(result[0]) == {'_key': '1', 'loc': [1, 1], 'dist': 0} # Test find_in_radius with bad collection with assert_raises(DocumentGetError) as err: @@ -1199,7 +1198,10 @@ def test_document_find_in_radius(col, bad_col): assert err.value.error_code in {11, 1228} -def test_document_find_in_box(col, bad_col, geo): +def test_document_find_in_box(col, bad_col, geo, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + doc1 = {'_key': '1', 'loc': [1, 1]} doc2 = {'_key': '2', 'loc': [1, 5]} doc3 = {'_key': '3', 'loc': [5, 1]} @@ -1545,7 +1547,7 @@ def test_document_get_many(col, bad_col, docs): assert clean_doc(result) == docs # Test get_many in empty collection - col.truncate() + empty_collection(col) assert col.get_many([]) == [] assert col.get_many(docs[:1]) == [] assert col.get_many(docs[:3]) == [] @@ -1655,74 +1657,77 @@ def test_document_keys(col, bad_col, docs): assert err.value.error_code in {11, 1228} -# def test_document_export(col, bad_col, docs): -# # Set up test documents -# col.insert_many(docs) -# -# # Test export with flush set to True and flush_wait set to 1 -# cursor = col.export(flush=True, flush_wait=1) -# assert clean_doc(cursor) == docs -# assert cursor.type == 'export' -# -# # Test export with count -# cursor = col.export(flush=False, count=True) -# assert cursor.count == len(docs) -# assert clean_doc(cursor) == docs -# -# # Test export with batch size -# cursor = col.export(flush=False, count=True, batch_size=1) -# assert cursor.count == len(docs) -# assert clean_doc(cursor) == docs -# -# # Test export with time-to-live -# cursor = col.export(flush=False, count=True, ttl=10) -# assert cursor.count == len(docs) -# assert clean_doc(cursor) == docs -# -# # Test export with filters -# cursor = col.export( -# count=True, -# flush=False, -# filter_fields=['text'], -# filter_type='exclude' -# ) -# assert cursor.count == len(docs) -# assert all(['text' not in d for d in cursor]) -# -# # Test export with a limit of 0 -# cursor = col.export(flush=False, count=True, limit=0) -# assert cursor.count == len(docs) -# assert clean_doc(cursor) == docs -# -# # Test export with a limit of 1 -# cursor = col.export(flush=False, count=True, limit=1) -# assert cursor.count == 1 -# assert len(list(cursor)) == 1 -# all([clean_doc(d) in docs for d in cursor]) -# -# # Test export with a limit of 3 -# cursor = col.export(flush=False, count=True, limit=3) -# assert cursor.count == 3 -# assert len(list(cursor)) == 3 -# all([clean_doc(d) in docs for d in cursor]) -# -# # Test export with bad database -# with assert_raises(DocumentGetError): -# bad_col.export() -# -# # Test closing export cursor -# cursor = col.export(flush=False, count=True, batch_size=1) -# assert cursor.close(ignore_missing=False) is True -# assert cursor.close(ignore_missing=True) is False -# -# assert clean_doc(cursor.next()) in docs -# with assert_raises(CursorNextError): -# cursor.next() -# with assert_raises(CursorCloseError): -# cursor.close(ignore_missing=False) -# -# cursor = col.export(flush=False, count=True) -# assert cursor.close(ignore_missing=True) is False +def test_document_export(col, bad_col, docs, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + + # Set up test documents + col.insert_many(docs) + + # Test export with flush set to True and flush_wait set to 1 + cursor = col.export(flush=True, flush_wait=1) + assert clean_doc(cursor) == docs + assert cursor.type == 'export' + + # Test export with count + cursor = col.export(flush=False, count=True) + assert cursor.count() == len(docs) + assert clean_doc(cursor) == docs + + # Test export with batch size + cursor = col.export(flush=False, count=True, batch_size=1) + assert cursor.count() == len(docs) + assert clean_doc(cursor) == docs + + # Test export with time-to-live + cursor = col.export(flush=False, count=True, ttl=10) + assert cursor.count() == len(docs) + assert clean_doc(cursor) == docs + + # Test export with filters + cursor = col.export( + count=True, + flush=False, + filter_fields=['text'], + filter_type='exclude' + ) + assert cursor.count() == len(docs) + assert all(['text' not in d for d in cursor]) + + # Test export with a limit of 0 + cursor = col.export(flush=False, count=True, limit=0) + assert cursor.count() == 0 + assert clean_doc(cursor) == [] + + # Test export with a limit of 1 + cursor = col.export(flush=False, count=True, limit=1) + assert cursor.count() == 1 + assert len(list(cursor)) == 1 + all([clean_doc(d) in docs for d in cursor]) + + # Test export with a limit of 3 + cursor = col.export(flush=False, count=True, limit=3) + assert cursor.count() == 3 + assert len(list(cursor)) == 3 + all([clean_doc(d) in docs for d in cursor]) + + # Test export with bad database + with assert_raises(DocumentGetError): + bad_col.export() + + # Test closing export cursor + cursor = col.export(flush=False, count=True, batch_size=1) + assert cursor.close(ignore_missing=False) is True + assert cursor.close(ignore_missing=True) is False + + assert clean_doc(cursor.next()) in docs + with assert_raises(CursorNextError): + cursor.next() + with assert_raises(CursorCloseError): + cursor.close(ignore_missing=False) + + cursor = col.export(flush=False, count=True) + assert cursor.close(ignore_missing=True) is None def test_document_random(col, bad_col, docs): @@ -1735,7 +1740,7 @@ def test_document_random(col, bad_col, docs): assert clean_doc(random_doc) in docs # Test random in empty collection - col.truncate() + empty_collection(col) for attempt in range(10): random_doc = col.random() assert random_doc is None @@ -1761,7 +1766,7 @@ def test_document_import_bulk(col, bad_col, docs): assert col[doc_key]['_key'] == doc_key assert col[doc_key]['val'] == doc['val'] assert col[doc_key]['loc'] == doc['loc'] - col.truncate() + empty_collection(col) # Test import bulk without details and with sync result = col.import_bulk(docs, details=False, sync=True) @@ -1789,7 +1794,7 @@ def test_document_import_bulk(col, bad_col, docs): assert result['empty'] == 0 assert result['updated'] == 0 assert result['ignored'] == 0 - col.truncate() + empty_collection(col) # Test import bulk with bad database with assert_raises(DocumentInsertError): @@ -1809,7 +1814,7 @@ def test_document_import_bulk(col, bad_col, docs): assert col[doc_key]['_key'] == doc_key assert col[doc_key]['val'] == doc['val'] assert col[doc_key]['loc'] == doc['loc'] - col.truncate() + empty_collection(col) # Test import bulk on_duplicate actions doc = docs[0] @@ -1851,7 +1856,7 @@ def test_document_import_bulk(col, bad_col, docs): assert col[doc['_key']]['foo'] == '2' assert col[doc['_key']]['bar'] == '3' - col.truncate() + empty_collection(col) col.insert(old_doc) result = col.import_bulk([new_doc], on_duplicate='replace', halt_on_error=False) @@ -1865,164 +1870,6 @@ def test_document_import_bulk(col, bad_col, docs): assert col[doc['_key']]['bar'] == '3' -@pytest.mark.skip(reason='broken in ArangoDB 3.4') -def test_document_edge(lecol, docs, edocs): - ecol = lecol # legacy edge collection - - # Test insert edge without "_from" and "_to" fields - with assert_raises(DocumentInsertError): - ecol.insert(docs[0]) - - # Test insert many edges without "_from" and "_to" fields - for result in ecol.insert_many(docs): - assert isinstance(result, DocumentInsertError) - - # Test update edge without "_from" and "_to" fields - with assert_raises(DocumentUpdateError): - ecol.update(docs[0]) - - # Test update many edges without "_from" and "_to" fields - for result in ecol.update_many(docs): - assert isinstance(result, DocumentUpdateError) - - # Test replace edge without "_from" and "_to" fields - with assert_raises(DocumentReplaceError): - ecol.replace(docs[0]) - - # Test replace many edges without "_from" and "_to" fields - for result in ecol.replace_many(docs): - assert isinstance(result, DocumentReplaceError) - - # Test edge document happy path - edoc = edocs[0] - - # Test insert edge - result = ecol.insert(edoc, return_new=True, sync=True) - assert len(ecol) == 1 - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == edoc['_key'] == ecol[edoc]['_key'] - assert result['new']['_from'] == edoc['_from'] == ecol[edoc]['_from'] - assert result['new']['_to'] == edoc['_to'] == ecol[edoc]['_to'] - - # Test update edge - new_edoc = edoc.copy() - new_edoc.update({'_from': 'foo', '_to': 'bar'}) - result = ecol.update(new_edoc, return_old=True, return_new=True) - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == new_edoc['_key'] - assert result['new']['_from'] == new_edoc['_from'] - assert result['new']['_to'] == new_edoc['_to'] - assert result['old']['_key'] == edoc['_key'] - assert result['old']['_from'] == edoc['_from'] - assert result['old']['_to'] == edoc['_to'] - assert ecol[edoc]['_key'] == edoc['_key'] - assert ecol[edoc]['_from'] == new_edoc['_from'] - assert ecol[edoc]['_to'] == new_edoc['_to'] - edoc = new_edoc - - # Test replace edge - new_edoc = edoc.copy() - new_edoc.update({'_from': 'baz', '_to': 'qux'}) - result = ecol.replace(new_edoc, return_old=True, return_new=True) - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == new_edoc['_key'] - assert result['new']['_from'] == new_edoc['_from'] - assert result['new']['_to'] == new_edoc['_to'] - assert result['old']['_key'] == edoc['_key'] - assert result['old']['_from'] == edoc['_from'] - assert result['old']['_to'] == edoc['_to'] - assert ecol[edoc]['_key'] == edoc['_key'] - assert ecol[edoc]['_from'] == new_edoc['_from'] - assert ecol[edoc]['_to'] == new_edoc['_to'] - edoc = new_edoc - - # Test delete edge - result = ecol.delete(edoc, return_old=True) - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['old']['_key'] == edoc['_key'] - assert result['old']['_from'] == edoc['_from'] - assert result['old']['_to'] == edoc['_to'] - assert edoc not in ecol - - # Test insert many edges - results = ecol.insert_many(edocs, return_new=True, sync=True) - for result, edoc in zip(results, edocs): - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == edoc['_key'] - assert result['new']['_from'] == edoc['_from'] - assert result['new']['_to'] == edoc['_to'] - assert ecol[edoc]['_key'] == edoc['_key'] - assert ecol[edoc]['_from'] == edoc['_from'] - assert ecol[edoc]['_to'] == edoc['_to'] - assert len(ecol) == 4 - - # Test update many edges - for edoc in edocs: - edoc['foo'] = 1 - results = ecol.update_many(edocs, return_new=True, sync=True) - for result, edoc in zip(results, edocs): - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == edoc['_key'] - assert result['new']['_from'] == edoc['_from'] - assert result['new']['_to'] == edoc['_to'] - assert result['new']['foo'] == 1 - assert ecol[edoc]['_key'] == edoc['_key'] - assert ecol[edoc]['_from'] == edoc['_from'] - assert ecol[edoc]['_to'] == edoc['_to'] - assert ecol[edoc]['foo'] == 1 - assert len(ecol) == 4 - - # Test replace many edges - for edoc in edocs: - edoc['bar'] = edoc.pop('foo') - results = ecol.replace_many(edocs, return_new=True, sync=True) - for result, edoc in zip(results, edocs): - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['new']['_key'] == edoc['_key'] - assert result['new']['_from'] == edoc['_from'] - assert result['new']['_to'] == edoc['_to'] - assert result['new']['bar'] == 1 - assert 'foo' not in result['new'] - assert ecol[edoc]['_key'] == edoc['_key'] - assert ecol[edoc]['_from'] == edoc['_from'] - assert ecol[edoc]['_to'] == edoc['_to'] - assert ecol[edoc]['bar'] == 1 - assert 'foo' not in ecol[edoc] - assert len(ecol) == 4 - - results = ecol.delete_many(edocs, return_old=True) - for result, edoc in zip(results, edocs): - assert result['_id'] == '{}/{}'.format(ecol.name, edoc['_key']) - assert result['_key'] == edoc['_key'] - assert result['old']['_key'] == edoc['_key'] - assert result['old']['_from'] == edoc['_from'] - assert result['old']['_to'] == edoc['_to'] - assert edoc not in ecol - assert edoc['_key'] not in ecol - assert len(ecol) == 0 - - # Test import bulk to_prefix and from_prefix - for doc in edocs: - doc['_from'] = 'foo' - doc['_to'] = 'bar' - result = ecol.import_bulk(edocs, from_prefix='from', to_prefix='to') - assert result['created'] == 4 - assert result['errors'] == 0 - assert result['empty'] == 0 - assert result['updated'] == 0 - assert result['ignored'] == 0 - for edoc in ecol: - assert edoc['_from'] == 'from/foo' - assert edoc['_to'] == 'to/bar' - - def test_document_management_via_db(db, col): doc1_id = col.name + '/foo' doc2_id = col.name + '/bar' diff --git a/tests/test_exception.py b/tests/test_exception.py index 406346ee..37f19a86 100644 --- a/tests/test_exception.py +++ b/tests/test_exception.py @@ -26,11 +26,9 @@ def test_server_error(client, col, docs): assert exc.source == 'server' assert exc.message == str(exc) assert exc.message.startswith('[HTTP 409][ERR 1210] unique constraint') - assert exc.url.startswith(client.base_url) assert exc.error_code == 1210 assert exc.http_method == 'post' assert exc.http_code == 409 - assert exc.http_headers['Server'] == 'ArangoDB' assert isinstance(exc.http_headers, CaseInsensitiveDict) resp = exc.response @@ -48,20 +46,17 @@ def test_server_error(client, col, docs): assert resp.method == 'post' assert resp.status_code == 409 assert resp.status_text == 'Conflict' + assert json.loads(resp.raw_body) == expected_body assert resp.headers == exc.http_headers - assert resp.url.startswith(client.base_url) req = exc.request assert isinstance(req, Request) assert req.headers['content-type'] == 'application/json' assert req.method == 'post' - assert req.read is None - assert req.write == col.name - assert req.command is None assert req.params['silent'] == 0 assert req.params['returnNew'] == 0 - assert req.data == json.dumps(document) + assert req.data == document assert req.endpoint.startswith('/_api/document/' + col.name) diff --git a/tests/test_foxx.py b/tests/test_foxx.py index f4d02aef..3f0f21f2 100644 --- a/tests/test_foxx.py +++ b/tests/test_foxx.py @@ -34,7 +34,6 @@ generate_service_mount ) -# noinspection PyUnresolvedReferences service_file = '/tmp/service.zip' @@ -324,11 +323,13 @@ def test_foxx_misc_functions(db, bad_db): result_string = db.foxx.run_tests( mount=service_mount, reporter='suite', - idiomatic=True + idiomatic=True, + name_filter='science' ) result_json = json.loads(result_string) assert 'stats' in result_json assert 'tests' in result_json + assert 'suites' in result_json result_string = db.foxx.run_tests( mount=service_mount, diff --git a/tests/test_graph.py b/tests/test_graph.py index e9cf79a1..5b1a4566 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -15,6 +15,7 @@ EdgeDefinitionCreateError, EdgeDefinitionDeleteError, EdgeDefinitionReplaceError, + EdgeListError, GraphListError, GraphCreateError, GraphDeleteError, @@ -22,11 +23,12 @@ GraphTraverseError, VertexCollectionCreateError, VertexCollectionDeleteError, - VertexCollectionListError, - EdgeListError) + VertexCollectionListError +) from tests.helpers import ( assert_raises, clean_doc, + empty_collection, extract, generate_col_name, generate_graph_name, @@ -219,28 +221,6 @@ def test_edge_definition_management(db, graph, bad_graph): assert not graph.has_edge_collection(ecol_name) assert not db.has_collection(ecol_name) - ecol = graph.create_edge_definition(ecol_name, [], []) - assert graph.has_edge_definition(ecol_name) - assert graph.has_edge_collection(ecol_name) - assert db.has_collection(ecol_name) - assert isinstance(ecol, EdgeCollection) - - ecol = graph.edge_collection(ecol_name) - assert ecol.name == ecol_name - assert ecol.name in repr(ecol) - assert ecol.graph == graph.name - assert { - 'edge_collection': ecol_name, - 'from_vertex_collections': [], - 'to_vertex_collections': [] - } in graph.edge_definitions() - assert ecol_name in extract('name', db.collections()) - - # Test create duplicate edge definition - with assert_raises(EdgeDefinitionCreateError) as err: - graph.create_edge_definition(ecol_name, [], []) - assert err.value.error_code == 1920 - # Test create edge definition with existing vertex collections fvcol_name = generate_col_name() tvcol_name = generate_col_name() @@ -251,6 +231,8 @@ def test_edge_definition_management(db, graph, bad_graph): to_vertex_collections=[tvcol_name] ) assert ecol.name == ecol_name + assert ecol.graph == graph.name + assert repr(ecol) == ''.format(ecol.name) assert { 'edge_collection': ecol_name, 'from_vertex_collections': [fvcol_name], @@ -262,6 +244,15 @@ def test_edge_definition_management(db, graph, bad_graph): assert fvcol_name in vertex_collections assert tvcol_name in vertex_collections + # Test create duplicate edge definition + with assert_raises(EdgeDefinitionCreateError) as err: + graph.create_edge_definition( + edge_collection=ecol_name, + from_vertex_collections=[fvcol_name], + to_vertex_collections=[tvcol_name] + ) + assert err.value.error_code == 1920 + # Test create edge definition with missing vertex collection bad_vcol_name = generate_col_name() ecol_name = generate_col_name() @@ -349,7 +340,7 @@ def test_vertex_management(fvcol, bad_fvcol, fvdocs): result = fvcol.insert({}) assert result['_key'] in fvcol assert len(fvcol) == 1 - fvcol.truncate() + empty_collection(fvcol) # Test insert vertex with ID vertex_id = fvcol.name + '/' + 'foo' @@ -357,7 +348,7 @@ def test_vertex_management(fvcol, bad_fvcol, fvdocs): assert 'foo' in fvcol assert vertex_id in fvcol assert len(fvcol) == 1 - fvcol.truncate() + empty_collection(fvcol) with assert_raises(DocumentParseError) as err: fvcol.insert({'_id': generate_col_name() + '/' + 'foo'}) @@ -591,7 +582,7 @@ def test_vertex_management(fvcol, bad_fvcol, fvdocs): assert fvcol[vertex] is None assert vertex not in fvcol assert len(fvcol) == 2 - fvcol.truncate() + empty_collection(fvcol) def test_vertex_management_via_graph(graph, fvcol): @@ -634,7 +625,7 @@ def test_edge_management(ecol, bad_ecol, edocs, fvcol, fvdocs, tvcol, tvdocs): result = ecol.insert({'_from': edge['_from'], '_to': edge['_to']}) assert result['_key'] in ecol assert len(ecol) == 1 - ecol.truncate() + empty_collection(ecol) # Test insert vertex with ID edge_id = ecol.name + '/' + 'foo' @@ -646,7 +637,7 @@ def test_edge_management(ecol, bad_ecol, edocs, fvcol, fvdocs, tvcol, tvdocs): assert 'foo' in ecol assert edge_id in ecol assert len(ecol) == 1 - ecol.truncate() + empty_collection(ecol) with assert_raises(DocumentParseError) as err: ecol.insert({ @@ -887,7 +878,7 @@ def test_edge_management(ecol, bad_ecol, edocs, fvcol, fvdocs, tvcol, tvdocs): if ecol.context != 'transaction': assert ecol[edge] is None assert edge not in ecol - ecol.truncate() + empty_collection(ecol) def test_vertex_edges(db, bad_db): @@ -960,7 +951,7 @@ def test_edge_management_via_graph(graph, ecol, fvcol, fvdocs, tvcol, tvdocs): fvcol.insert(vertex) for vertex in tvdocs: tvcol.insert(vertex) - ecol.truncate() + empty_collection(ecol) # Get a random "from" vertex from_vertex = fvcol.random() diff --git a/tests/test_index.py b/tests/test_index.py index dc950883..d04d1459 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -9,55 +9,61 @@ from tests.helpers import assert_raises, extract -def test_list_indexes(col, bad_col): - expected_index = { - 'id': '0', - 'selectivity': 1, - 'sparse': False, - 'type': 'primary', - 'fields': ['_key'], - 'unique': True - } - indexes = col.indexes() +def test_list_indexes(icol, bad_col): + indexes = icol.indexes() assert isinstance(indexes, list) - assert expected_index in indexes + assert len(indexes) > 0 + assert 'id' in indexes[0] + assert 'type' in indexes[0] + assert 'fields' in indexes[0] + assert 'selectivity' in indexes[0] + assert 'sparse' in indexes[0] + assert 'unique' in indexes[0] with assert_raises(IndexListError) as err: bad_col.indexes() assert err.value.error_code in {11, 1228} -def test_add_hash_index(col): +def test_add_hash_index(icol): + icol = icol + fields = ['attr1', 'attr2'] - result = col.add_hash_index( + result = icol.add_hash_index( fields=fields, unique=True, sparse=True, - deduplicate=True + deduplicate=True, + name='hash_index', + in_background=False ) expected_index = { - 'selectivity': 1, 'sparse': True, 'type': 'hash', 'fields': ['attr1', 'attr2'], 'unique': True, - 'deduplicate': True + 'deduplicate': True, + 'name': 'hash_index' } for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) + # Clean up the index + icol.delete_index(result['id']) -def test_add_skiplist_index(col): + +def test_add_skiplist_index(icol): fields = ['attr1', 'attr2'] - result = col.add_skiplist_index( + result = icol.add_skiplist_index( fields=fields, unique=True, sparse=True, - deduplicate=True + deduplicate=True, + name='skiplist_index', + in_background=False ) expected_index = { @@ -65,20 +71,25 @@ def test_add_skiplist_index(col): 'type': 'skiplist', 'fields': ['attr1', 'attr2'], 'unique': True, - 'deduplicate': True + 'deduplicate': True, + 'name': 'skiplist_index' } for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) + + # Clean up the index + icol.delete_index(result['id']) -def test_add_geo_index(col): +def test_add_geo_index(icol): # Test add geo index with one attribute - result = col.add_geo_index( + result = icol.add_geo_index( fields=['attr1'], - ordered=False + ordered=False, + name='geo_index', + in_background=True ) expected_index = { @@ -87,15 +98,15 @@ def test_add_geo_index(col): 'fields': ['attr1'], 'unique': False, 'geo_json': False, + 'name': 'geo_index' } for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) # Test add geo index with two attributes - result = col.add_geo_index( + result = icol.add_geo_index( fields=['attr1', 'attr2'], ordered=False, ) @@ -108,20 +119,24 @@ def test_add_geo_index(col): for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) # Test add geo index with more than two attributes (should fail) with assert_raises(IndexCreateError) as err: - col.add_geo_index(fields=['attr1', 'attr2', 'attr3']) + icol.add_geo_index(fields=['attr1', 'attr2', 'attr3']) assert err.value.error_code == 10 + # Clean up the index + icol.delete_index(result['id']) -def test_add_fulltext_index(col): + +def test_add_fulltext_index(icol): # Test add fulltext index with one attributes - result = col.add_fulltext_index( + result = icol.add_fulltext_index( fields=['attr1'], min_length=10, + name='fulltext_index', + in_background=True ) expected_index = { 'sparse': True, @@ -129,61 +144,92 @@ def test_add_fulltext_index(col): 'fields': ['attr1'], 'min_length': 10, 'unique': False, + 'name': 'fulltext_index' } for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) # Test add fulltext index with two attributes (should fail) with assert_raises(IndexCreateError) as err: - col.add_fulltext_index(fields=['attr1', 'attr2']) + icol.add_fulltext_index(fields=['attr1', 'attr2']) assert err.value.error_code == 10 + # Clean up the index + icol.delete_index(result['id']) + -def test_add_persistent_index(col): +def test_add_persistent_index(icol): # Test add persistent index with two attributes - result = col.add_persistent_index( + result = icol.add_persistent_index( fields=['attr1', 'attr2'], unique=True, sparse=True, + name='persistent_index', + in_background=True ) expected_index = { 'sparse': True, 'type': 'persistent', 'fields': ['attr1', 'attr2'], 'unique': True, + 'name': 'persistent_index' } for key, value in expected_index.items(): assert result[key] == value - result.pop('new', None) - assert result in col.indexes() + assert result['id'] in extract('id', icol.indexes()) + + # Clean up the index + icol.delete_index(result['id']) + + +def test_add_ttl_index(icol): + # Test add persistent index with two attributes + result = icol.add_ttl_index( + fields=['attr1'], + expiry_time=1000, + name='ttl_index', + in_background=True + ) + expected_index = { + 'type': 'ttl', + 'fields': ['attr1'], + 'expiry_time': 1000, + 'name': 'ttl_index' + } + for key, value in expected_index.items(): + assert result[key] == value + + assert result['id'] in extract('id', icol.indexes()) + + # Clean up the index + icol.delete_index(result['id']) -def test_delete_index(col, bad_col): - old_indexes = set(extract('id', col.indexes())) - col.add_hash_index(['attr3', 'attr4'], unique=True) - col.add_skiplist_index(['attr3', 'attr4'], unique=True) - col.add_fulltext_index(fields=['attr3'], min_length=10) +def test_delete_index(icol, bad_col): + old_indexes = set(extract('id', icol.indexes())) + icol.add_hash_index(['attr3', 'attr4'], unique=True) + icol.add_skiplist_index(['attr3', 'attr4'], unique=True) + icol.add_fulltext_index(fields=['attr3'], min_length=10) - new_indexes = set(extract('id', col.indexes())) + new_indexes = set(extract('id', icol.indexes())) assert new_indexes.issuperset(old_indexes) indexes_to_delete = new_indexes - old_indexes for index_id in indexes_to_delete: - assert col.delete_index(index_id) is True + assert icol.delete_index(index_id) is True - new_indexes = set(extract('id', col.indexes())) + new_indexes = set(extract('id', icol.indexes())) assert new_indexes == old_indexes # Test delete missing indexes for index_id in indexes_to_delete: - assert col.delete_index(index_id, ignore_missing=True) is False + assert icol.delete_index(index_id, ignore_missing=True) is False for index_id in indexes_to_delete: with assert_raises(IndexDeleteError) as err: - col.delete_index(index_id, ignore_missing=False) + icol.delete_index(index_id, ignore_missing=False) assert err.value.error_code == 1212 # Test delete indexes with bad collection @@ -193,9 +239,9 @@ def test_delete_index(col, bad_col): assert err.value.error_code in {11, 1228} -def test_load_indexes(col, bad_col): +def test_load_indexes(icol, bad_col): # Test load indexes - assert col.load_indexes() is True + assert icol.load_indexes() is True # Test load indexes with bad collection with assert_raises(IndexLoadError) as err: diff --git a/tests/test_permission.py b/tests/test_permission.py index 03a713ce..99dabdbf 100644 --- a/tests/test_permission.py +++ b/tests/test_permission.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals +import pytest + from arango.exceptions import ( CollectionCreateError, CollectionListError, @@ -20,7 +22,10 @@ ) -def test_permission_management(client, sys_db, bad_db): +def test_permission_management(client, sys_db, bad_db, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + username = generate_username() password = generate_string() db_name = generate_db_name() @@ -49,19 +54,13 @@ def test_permission_management(client, sys_db, bad_db): assert err.value.error_code in {11, 1228} # The user should not have read and write permissions - assert sys_db.permission(username, db_name) == 'none' - assert sys_db.permission(username, db_name, col_name_1) == 'none' - with assert_raises(CollectionCreateError) as err: - db.create_collection(col_name_1) - assert err.value.http_code == 401 - with assert_raises(CollectionListError) as err: - db.collections() - assert err.value.http_code == 401 + assert sys_db.permission(username, db_name) == 'rw' + assert sys_db.permission(username, db_name, col_name_1) == 'rw' # Test update permission (database level) with bad database with assert_raises(PermissionUpdateError): bad_db.update_permission(username, 'ro', db_name) - assert sys_db.permission(username, db_name) == 'none' + assert sys_db.permission(username, db_name) == 'rw' # Test update permission (database level) to read only and verify access assert sys_db.update_permission(username, 'ro', db_name) is True diff --git a/tests/test_pregel.py b/tests/test_pregel.py index 85ed902c..7b3af293 100644 --- a/tests/test_pregel.py +++ b/tests/test_pregel.py @@ -1,5 +1,8 @@ from __future__ import absolute_import, unicode_literals +import time + +import pytest from six import string_types from arango.exceptions import ( @@ -20,7 +23,10 @@ def test_pregel_attributes(db, username): assert repr(db.pregel) == ''.format(db.name) -def test_pregel_management(db, graph): +def test_pregel_management(db, graph, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + # Test create pregel job job_id = db.pregel.create_job( graph.name, @@ -37,7 +43,7 @@ def test_pregel_management(db, graph): # Test create pregel job with unsupported algorithm with assert_raises(PregelJobCreateError) as err: db.pregel.create_job(graph.name, 'invalid') - assert err.value.error_code in {4, 10} + assert err.value.error_code in {4, 10, 1600} # Test get existing pregel job job = db.pregel.job(job_id) @@ -46,15 +52,16 @@ def test_pregel_management(db, graph): assert isinstance(job['gss'], int) assert isinstance(job['received_count'], int) assert isinstance(job['send_count'], int) - assert isinstance(job['total_runtime'], float) + assert 'total_runtime' in job # Test delete existing pregel job assert db.pregel.delete_job(job_id) is True + time.sleep(0.2) with assert_raises(PregelJobGetError) as err: db.pregel.job(job_id) - assert err.value.error_code in {4, 10} + assert err.value.error_code in {4, 10, 1600} # Test delete missing pregel job with assert_raises(PregelJobDeleteError) as err: db.pregel.delete_job(generate_string()) - assert err.value.error_code in {4, 10} + assert err.value.error_code in {4, 10, 1600} diff --git a/tests/test_request.py b/tests/test_request.py index 6d973864..57291648 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -10,12 +10,6 @@ def test_request_no_data(): params={'bool': True}, headers={'foo': 'bar'} ) - assert str(request) == '\r\n'.join([ - 'post /_api/test?bool=1 HTTP/1.1', - 'charset: utf-8', - 'content-type: application/json', - 'foo: bar', - ]) assert request.method == 'post' assert request.endpoint == '/_api/test' assert request.params == {'bool': 1} @@ -25,9 +19,6 @@ def test_request_no_data(): 'foo': 'bar', } assert request.data is None - assert request.command is None - assert request.read is None - assert request.write is None def test_request_string_data(): @@ -38,13 +29,6 @@ def test_request_string_data(): headers={'foo': 'bar'}, data='test' ) - assert str(request) == '\r\n'.join([ - 'post /_api/test?bool=1 HTTP/1.1', - 'charset: utf-8', - 'content-type: application/json', - 'foo: bar', - '\r\ntest', - ]) assert request.method == 'post' assert request.endpoint == '/_api/test' assert request.params == {'bool': 1} @@ -54,9 +38,6 @@ def test_request_string_data(): 'foo': 'bar', } assert request.data == 'test' - assert request.command is None - assert request.read is None - assert request.write is None def test_request_json_data(): @@ -67,13 +48,6 @@ def test_request_json_data(): headers={'foo': 'bar'}, data={'baz': 'qux'} ) - assert str(request) == '\r\n'.join([ - 'post /_api/test?bool=1 HTTP/1.1', - 'charset: utf-8', - 'content-type: application/json', - 'foo: bar', - '\r\n{"baz": "qux"}', - ]) assert request.method == 'post' assert request.endpoint == '/_api/test' assert request.params == {'bool': 1} @@ -82,10 +56,7 @@ def test_request_json_data(): 'content-type': 'application/json', 'foo': 'bar', } - assert request.data == '{"baz": "qux"}' - assert request.command is None - assert request.read is None - assert request.write is None + assert request.data == {'baz': 'qux'} def test_request_transaction_data(): @@ -95,17 +66,7 @@ def test_request_transaction_data(): params={'bool': True}, headers={'foo': 'bar'}, data={'baz': 'qux'}, - command='return 1', - read='one', - write='two', ) - assert str(request) == '\r\n'.join([ - 'post /_api/test?bool=1 HTTP/1.1', - 'charset: utf-8', - 'content-type: application/json', - 'foo: bar', - '\r\n{"baz": "qux"}', - ]) assert request.method == 'post' assert request.endpoint == '/_api/test' assert request.params == {'bool': 1} @@ -114,7 +75,4 @@ def test_request_transaction_data(): 'content-type': 'application/json', 'foo': 'bar', } - assert request.data == '{"baz": "qux"}' - assert request.command == 'return 1' - assert request.read == 'one' - assert request.write == 'two' + assert request.data == {'baz': 'qux'} diff --git a/tests/test_resolver.py b/tests/test_resolver.py new file mode 100644 index 00000000..73d74248 --- /dev/null +++ b/tests/test_resolver.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import, unicode_literals + +from arango.resolver import ( + SingleHostResolver, + RandomHostResolver, + RoundRobinHostResolver +) + + +def test_resolver_single_host(): + resolver = SingleHostResolver() + for _ in range(20): + assert resolver.get_host_index() == 0 + + +def test_resolver_random_host(): + resolver = RandomHostResolver(10) + for _ in range(20): + assert 0 <= resolver.get_host_index() < 10 + + +def test_resolver_round_robin(): + resolver = RoundRobinHostResolver(10) + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 3 + assert resolver.get_host_index() == 4 + assert resolver.get_host_index() == 5 + assert resolver.get_host_index() == 6 + assert resolver.get_host_index() == 7 + assert resolver.get_host_index() == 8 + assert resolver.get_host_index() == 9 + assert resolver.get_host_index() == 0 diff --git a/tests/test_response.py b/tests/test_response.py index 1d2d9b78..2e7176f4 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -5,7 +5,7 @@ from arango.response import Response -def test_response(): +def test_response(conn): response = Response( method='get', url='test_url', @@ -14,15 +14,18 @@ def test_response(): status_code=200, raw_body='true', ) + conn.prep_response(response) + assert response.method == 'get' assert response.url == 'test_url' assert response.headers == {'foo': 'bar'} assert response.status_code == 200 assert response.status_text == 'baz' - assert response.body is True assert response.raw_body == 'true' + assert response.body is True assert response.error_code is None assert response.error_message is None + assert response.is_success is True test_body = '{"errorNum": 1, "errorMessage": "qux"}' response = Response( @@ -33,30 +36,15 @@ def test_response(): status_code=200, raw_body=test_body, ) + conn.prep_response(response) + assert response.method == 'get' assert response.url == 'test_url' assert response.headers == {'foo': 'bar'} assert response.status_code == 200 assert response.status_text == 'baz' - assert response.body == {'errorNum': 1, 'errorMessage': 'qux'} assert response.raw_body == test_body + assert response.body == {'errorMessage': 'qux', 'errorNum': 1} assert response.error_code == 1 assert response.error_message == 'qux' - - response = Response( - method='get', - url='test_url', - headers=CaseInsensitiveDict({'foo': 'bar'}), - status_text='baz', - status_code=200, - raw_body='invalid', - ) - assert response.method == 'get' - assert response.url == 'test_url' - assert response.headers == {'foo': 'bar'} - assert response.status_code == 200 - assert response.status_text == 'baz' - assert response.body == 'invalid' - assert response.raw_body == 'invalid' - assert response.error_code is None - assert response.error_message is None + assert response.is_success is False diff --git a/tests/test_task.py b/tests/test_task.py index 4f2a04c4..1910d300 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -68,7 +68,6 @@ def test_task_management(sys_db, db, bad_db): # Test list tasks for task in sys_db.tasks(): - assert task['database'] in db.databases() assert task['type'] in {'periodic', 'timed'} assert isinstance(task['id'], string_types) assert isinstance(task['name'], string_types) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index fa8e897d..b1b21ac8 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -1,251 +1,139 @@ from __future__ import absolute_import, unicode_literals import pytest -from six import string_types from arango.database import TransactionDatabase from arango.exceptions import ( - TransactionStateError, TransactionExecuteError, - TransactionJobResultError + TransactionInitError, + TransactionStatusError, + TransactionCommitError, + TransactionAbortError ) -from arango.job import TransactionJob -from tests.helpers import clean_doc, extract, generate_string +from tests.helpers import extract -# noinspection PyUnresolvedReferences -def test_transaction_wrapper_attributes(db, col, username): - txn_db = db.begin_transaction(timeout=100, sync=True) - assert txn_db._executor._sync is True - assert txn_db._executor._timeout == 100 +def test_transaction_execute_raw(db, col, docs): + # Test execute raw transaction + doc = docs[0] + key = doc['_key'] + result = db.execute_transaction( + command=''' + function (params) {{ + var db = require('internal').db; + db.{col}.save({{'_key': params.key, 'val': 1}}); + return true; + }} + '''.format(col=col.name), + params={'key': key}, + write=[col.name], + read=[col.name], + sync=False, + timeout=1000, + max_size=100000, + allow_implicit=True, + intermediate_commit_count=10, + intermediate_commit_size=10000 + ) + assert result is True + assert doc in col and col[key]['val'] == 1 + + # Test execute invalid transaction + with pytest.raises(TransactionExecuteError) as err: + db.execute_transaction(command='INVALID COMMAND') + assert err.value.error_code == 10 + + +def test_transaction_init(db, bad_db, col, username): + txn_db = db.begin_transaction() + assert isinstance(txn_db, TransactionDatabase) assert txn_db.username == username assert txn_db.context == 'transaction' assert txn_db.db_name == db.name assert txn_db.name == db.name + assert txn_db.transaction_id is not None assert repr(txn_db) == ''.format(db.name) txn_col = txn_db.collection(col.name) assert txn_col.username == username assert txn_col.context == 'transaction' assert txn_col.db_name == db.name - assert txn_col.name == col.name txn_aql = txn_db.aql assert txn_aql.username == username assert txn_aql.context == 'transaction' assert txn_aql.db_name == db.name - job = txn_col.get(generate_string()) - assert isinstance(job, TransactionJob) - assert isinstance(job.id, string_types) - assert repr(job) == ''.format(job.id) - - -def test_transaction_execute_without_result(db, col, docs): - with db.begin_transaction(return_result=False) as txn_db: - txn_col = txn_db.collection(col.name) - - # Ensure that no jobs are returned - assert txn_col.insert(docs[0]) is None - assert txn_col.delete(docs[0]) is None - assert txn_col.insert(docs[1]) is None - assert txn_col.delete(docs[1]) is None - assert txn_col.insert(docs[2]) is None - assert txn_col.get(docs[2]) is None - assert txn_db.queued_jobs() is None - - # Ensure that the operations went through - assert txn_db.queued_jobs() is None - assert extract('_key', col.all()) == [docs[2]['_key']] - - -def test_transaction_execute_with_result(db, col, docs): - with db.begin_transaction(return_result=True) as txn_db: - txn_col = txn_db.collection(col.name) - job1 = txn_col.insert(docs[0]) - job2 = txn_col.insert(docs[1]) - job3 = txn_col.get(docs[1]) - jobs = txn_db.queued_jobs() - assert jobs == [job1, job2, job3] - assert all(job.status() == 'pending' for job in jobs) - - assert txn_db.queued_jobs() == [job1, job2, job3] - assert all(job.status() == 'done' for job in txn_db.queued_jobs()) - assert extract('_key', col.all()) == extract('_key', docs[:2]) - - # Test successful results - assert job1.result()['_key'] == docs[0]['_key'] - assert job2.result()['_key'] == docs[1]['_key'] - assert job3.result()['_key'] == docs[1]['_key'] - - -def test_transaction_execute_aql(db, col, docs): - with db.begin_transaction( - return_result=True, read=[col.name], write=[col.name]) as txn_db: - job1 = txn_db.aql.execute( - 'INSERT @data IN @@collection', - bind_vars={'data': docs[0], '@collection': col.name}) - job2 = txn_db.aql.execute( - 'INSERT @data IN @@collection', - bind_vars={'data': docs[1], '@collection': col.name}) - job3 = txn_db.aql.execute( - 'RETURN DOCUMENT(@@collection, @key)', - bind_vars={'key': docs[1]['_key'], '@collection': col.name}) - jobs = txn_db.queued_jobs() - assert jobs == [job1, job2, job3] - assert all(job.status() == 'pending' for job in jobs) - - assert txn_db.queued_jobs() == [job1, job2, job3] - assert all(job.status() == 'done' for job in txn_db.queued_jobs()) - assert extract('_key', col.all()) == extract('_key', docs[:2]) - - # Test successful results - assert extract('_key', job3.result()) == [docs[1]['_key']] - - -def test_transaction_execute_aql_string_form(db, col, docs): - with db.begin_transaction( - return_result=True, read=col.name, write=col.name) as txn_db: - job1 = txn_db.aql.execute( - 'INSERT @data IN @@collection', - bind_vars={'data': docs[0], '@collection': col.name}) - job2 = txn_db.aql.execute( - 'INSERT @data IN @@collection', - bind_vars={'data': docs[1], '@collection': col.name}) - job3 = txn_db.aql.execute( - 'RETURN DOCUMENT(@@collection, @key)', - bind_vars={'key': docs[1]['_key'], '@collection': col.name}) - jobs = txn_db.queued_jobs() - assert jobs == [job1, job2, job3] - assert all(job.status() == 'pending' for job in jobs) - - assert txn_db.queued_jobs() == [job1, job2, job3] - assert all(job.status() == 'done' for job in txn_db.queued_jobs()) - assert extract('_key', col.all()) == extract('_key', docs[:2]) - - # Test successful results - assert extract('_key', job3.result()) == [docs[1]['_key']] - - -def test_transaction_execute_error_in_result(db, col, docs): - txn_db = db.begin_transaction(timeout=100, sync=True) - txn_col = txn_db.collection(col.name) - job1 = txn_col.insert(docs[0]) - job2 = txn_col.insert(docs[1]) - job3 = txn_col.insert(docs[1]) # duplicate - - with pytest.raises(TransactionExecuteError) as err: - txn_db.commit() - assert err.value.error_code == 1210 - - jobs = [job1, job2, job3] - assert txn_db.queued_jobs() == jobs - assert all(job.status() == 'pending' for job in jobs) - + with pytest.raises(TransactionInitError) as err: + bad_db.begin_transaction() + assert err.value.error_code in {11, 1228} -def test_transaction_empty_commit(db): - txn_db = db.begin_transaction(return_result=True) - assert list(txn_db.commit()) == [] - txn_db = db.begin_transaction(return_result=False) - assert txn_db.commit() is None - - -def test_transaction_double_commit(db, col, docs): - txn_db = db.begin_transaction() - job = txn_db.collection(col.name).insert(docs[0]) +def test_transaction_status(db, col, docs): + txn_db = db.begin_transaction(read=col.name) + assert txn_db.transaction_status() == 'running' - # Test first commit - assert txn_db.commit() == [job] - assert job.status() == 'done' - assert len(col) == 1 - assert clean_doc(col.random()) == docs[0] + txn_db.commit_transaction() + assert txn_db.transaction_status() == 'committed' - # Test second commit which should fail - with pytest.raises(TransactionStateError) as err: - txn_db.commit() - assert 'already committed' in str(err.value) - assert job.status() == 'done' - assert len(col) == 1 - assert clean_doc(col.random()) == docs[0] + txn_db = db.begin_transaction(read=col.name) + assert txn_db.transaction_status() == 'running' + txn_db.abort_transaction() + assert txn_db.transaction_status() == 'aborted' -def test_transaction_action_after_commit(db, col): - with db.begin_transaction() as txn_db: - txn_db.collection(col.name).insert({}) + # Test transaction_status with an illegal transaction ID + txn_db._executor._id = 'illegal' + with pytest.raises(TransactionStatusError) as err: + txn_db.transaction_status() + assert err.value.error_code in {10, 1655} - # Test insert after the transaction has been committed - with pytest.raises(TransactionStateError) as err: - txn_db.collection(col.name).insert({}) - assert 'already committed' in str(err.value) - assert len(col) == 1 +def test_transaction_commit(db, col, docs): + txn_db = db.begin_transaction( + read=col.name, + write=col.name, + exclusive=[], + sync=True, + allow_implicit=False, + lock_timeout=1000, + max_size=10000 + ) + txn_col = txn_db.collection(col.name) -def test_transaction_method_not_allowed(db): - with pytest.raises(TransactionStateError) as err: - txn_db = db.begin_transaction() - txn_db.aql.functions() - assert str(err.value) == 'action not allowed in transaction' - - with pytest.raises(TransactionStateError) as err: - with db.begin_transaction() as txn_db: - txn_db.aql.functions() - assert str(err.value) == 'action not allowed in transaction' - - -def test_transaction_execute_error(bad_db, col, docs): - txn_db = bad_db.begin_transaction(return_result=True) - job = txn_db.collection(col.name).insert_many(docs) - - # Test transaction execute with bad database - with pytest.raises(TransactionExecuteError): - txn_db.commit() - assert len(col) == 0 - assert job.status() == 'pending' + assert '_rev' in txn_col.insert(docs[0]) + assert '_rev' in txn_col.delete(docs[0]) + assert '_rev' in txn_col.insert(docs[1]) + assert '_rev' in txn_col.delete(docs[1]) + assert '_rev' in txn_col.insert(docs[2]) + txn_db.commit_transaction() + assert extract('_key', col.all()) == [docs[2]['_key']] + assert txn_db.transaction_status() == 'committed' -def test_transaction_job_result_not_ready(db, col, docs): - txn_db = db.begin_transaction(return_result=True) - job = txn_db.collection(col.name).insert_many(docs) + # Test commit_transaction with an illegal transaction ID + txn_db._executor._id = 'illegal' + with pytest.raises(TransactionCommitError) as err: + txn_db.commit_transaction() + assert err.value.error_code in {10, 1655} - # Test get job result before commit - with pytest.raises(TransactionJobResultError) as err: - job.result() - assert str(err.value) == 'result not available yet' - # Test commit to make sure it still works after the errors - assert list(txn_db.commit()) == [job] - assert len(job.result()) == len(docs) - assert extract('_key', col.all()) == extract('_key', docs) +def test_transaction_abort(db, col, docs): + txn_db = db.begin_transaction(write=col.name) + txn_col = txn_db.collection(col.name) + assert '_rev' in txn_col.insert(docs[0]) + assert '_rev' in txn_col.delete(docs[0]) + assert '_rev' in txn_col.insert(docs[1]) + assert '_rev' in txn_col.delete(docs[1]) + assert '_rev' in txn_col.insert(docs[2]) + txn_db.abort_transaction() -def test_transaction_execute_raw(db, col, docs): - # Test execute raw transaction - doc = docs[0] - key = doc['_key'] - result = db.execute_transaction( - command=''' - function (params) {{ - var db = require('internal').db; - db.{col}.save({{'_key': params.key, 'val': 1}}); - return true; - }} - '''.format(col=col.name), - params={'key': key}, - write=[col.name], - read=[col.name], - sync=False, - timeout=1000, - max_size=100000, - allow_implicit=True, - intermediate_commit_count=10, - intermediate_commit_size=10000 - ) - assert result is True - assert doc in col and col[key]['val'] == 1 + assert extract('_key', col.all()) == [] + assert txn_db.transaction_status() == 'aborted' - # Test execute invalid transaction - with pytest.raises(TransactionExecuteError) as err: - db.execute_transaction(command='INVALID COMMAND') - assert err.value.error_code == 10 + txn_db._executor._id = 'illegal' + with pytest.raises(TransactionAbortError) as err: + txn_db.abort_transaction() + assert err.value.error_code in {10, 1655} diff --git a/tests/test_user.py b/tests/test_user.py index bc33be3b..72155792 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, unicode_literals +import pytest from six import string_types from arango.exceptions import ( @@ -120,7 +121,10 @@ def test_user_management(sys_db, bad_db): assert sys_db.delete_user(username, ignore_missing=True) is False -def test_user_change_password(client, sys_db): +def test_user_change_password(client, sys_db, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + username = generate_username() password1 = generate_string() password2 = generate_string() @@ -152,7 +156,10 @@ def test_user_change_password(client, sys_db): assert err.value.http_code == 401 -def test_user_create_with_new_database(client, sys_db): +def test_user_create_with_new_database(client, sys_db, cluster): + if cluster: + pytest.skip('Not tested in a cluster setup') + db_name = generate_db_name() username1 = generate_username() diff --git a/tests/test_view.py b/tests/test_view.py index a0f04c38..985986f8 100644 --- a/tests/test_view.py +++ b/tests/test_view.py @@ -12,17 +12,17 @@ from tests.helpers import assert_raises, generate_view_name -def test_view_management(db, bad_db): +def test_view_management(db, bad_db, cluster): # Test create view view_name = generate_view_name() bad_view_name = generate_view_name() view_type = 'arangosearch' - view_properties = { - 'consolidationIntervalMsec': 50000, - # 'consolidationPolicy': {'segmentThreshold': 200} - } - result = db.create_view(view_name, view_type, view_properties) + result = db.create_view( + view_name, + view_type, + {'consolidationIntervalMsec': 50000} + ) assert 'id' in result assert result['name'] == view_name assert result['type'] == view_type @@ -31,7 +31,11 @@ def test_view_management(db, bad_db): # Test create duplicate view with assert_raises(ViewCreateError) as err: - db.create_view(view_name, view_type, view_properties) + db.create_view( + view_name, + view_type, + {'consolidationIntervalMsec': 50000} + ) assert err.value.error_code == 1207 # Test list views @@ -42,7 +46,7 @@ def test_view_management(db, bad_db): assert view['name'] == view_name assert view['type'] == view_type - # Test list view with bad database + # Test list views with bad database with assert_raises(ViewListError) as err: bad_db.views() assert err.value.error_code in {11, 1228} @@ -66,7 +70,7 @@ def test_view_management(db, bad_db): assert view['type'] == view_type assert view['consolidationIntervalMsec'] == 70000 - # Test update with bad database + # Test update view with bad database with assert_raises(ViewUpdateError) as err: bad_db.update_view(view_name, {'consolidationIntervalMsec': 80000}) assert err.value.error_code in {11, 1228} @@ -78,24 +82,27 @@ def test_view_management(db, bad_db): assert view['type'] == view_type assert view['consolidationIntervalMsec'] == 40000 - # Test replace with bad database + # Test replace view with bad database with assert_raises(ViewReplaceError) as err: bad_db.replace_view(view_name, {'consolidationIntervalMsec': 7000}) assert err.value.error_code in {11, 1228} - # Test rename view - new_view_name = generate_view_name() - assert db.rename_view(view_name, new_view_name) is True - result = db.views() - assert len(result) == 1 - view = result[0] - assert view['id'] == view_id - assert view['name'] == new_view_name - - # Test rename missing view - with assert_raises(ViewRenameError) as err: - db.rename_view(bad_view_name, view_name) - assert err.value.error_code == 1203 + if cluster: + new_view_name = view_name + else: + # Test rename view + new_view_name = generate_view_name() + assert db.rename_view(view_name, new_view_name) is True + result = db.views() + assert len(result) == 1 + view = result[0] + assert view['id'] == view_id + assert view['name'] == new_view_name + + # Test rename missing view + with assert_raises(ViewRenameError) as err: + db.rename_view(bad_view_name, view_name) + assert err.value.error_code == 1203 # Test delete view assert db.delete_view(new_view_name) is True @@ -108,3 +115,71 @@ def test_view_management(db, bad_db): # Test delete missing view with ignore_missing set to True assert db.delete_view(view_name, ignore_missing=True) is False + + +def test_arangosearch_view_management(db, bad_db): + # Test create arangosearch view + view_name = generate_view_name() + result = db.create_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 50000} + ) + assert 'id' in result + assert result['name'] == view_name + assert result['type'].lower() == 'arangosearch' + assert result['consolidationIntervalMsec'] == 50000 + view_id = result['id'] + + # Test create duplicate arangosearch view + with assert_raises(ViewCreateError) as err: + db.create_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 50000} + ) + assert err.value.error_code == 1207 + + result = db.views() + assert len(result) == 1 + view = result[0] + assert view['id'] == view_id + assert view['name'] == view_name + assert view['type'] == 'arangosearch' + + # Test update arangosearch view + view = db.update_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 70000} + ) + assert view['id'] == view_id + assert view['name'] == view_name + assert view['type'].lower() == 'arangosearch' + assert view['consolidationIntervalMsec'] == 70000 + + # Test update arangosearch view with bad database + with assert_raises(ViewUpdateError) as err: + bad_db.update_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 70000} + ) + assert err.value.error_code in {11, 1228} + + # Test replace arangosearch view + view = db.replace_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 40000} + ) + assert view['id'] == view_id + assert view['name'] == view_name + assert view['type'] == 'arangosearch' + assert view['consolidationIntervalMsec'] == 40000 + + # Test replace arangosearch with bad database + with assert_raises(ViewReplaceError) as err: + bad_db.replace_arangosearch_view( + view_name, + {'consolidationIntervalMsec': 70000} + ) + assert err.value.error_code in {11, 1228} + + # Test delete arangosearch view + assert db.delete_view(view_name, ignore_missing=False) is True