From d5175e5057eb4365949fb141ff3c1c3859912f44 Mon Sep 17 00:00:00 2001 From: sanderland <48946947+sanderland@users.noreply.github.com> Date: Wed, 28 Aug 2019 14:49:50 +0200 Subject: [PATCH] Relationships api (#514) * relationships sdk * openapi generator fix --- CHANGELOG.md | 1 + cognite/client/_api/relationships.py | 171 +++++++++++++++++ cognite/client/data_classes/__init__.py | 1 + cognite/client/data_classes/relationships.py | 79 ++++++++ cognite/client/experimental.py | 2 + docs/source/cognite.rst | 32 ++++ generate_from_spec.py | 2 +- openapi/generator.py | 4 +- tests/conftest.py | 3 + .../tests_unit/test_api/test_relationships.py | 181 ++++++++++++++++++ tests/tests_unit/test_docstring_examples.py | 17 +- 11 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 cognite/client/_api/relationships.py create mode 100644 cognite/client/data_classes/relationships.py create mode 100644 tests/tests_unit/test_api/test_relationships.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 771c0d99cd..139849796b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ Changes are grouped as follows ### Added - `limit` parameter on sequence data retrieval. +- Support for relationships exposed through experimental client. - `end` parameter of sequence.data retrieval and range delete accepts -1 to indicate last index of sequence. ### Fixed diff --git a/cognite/client/_api/relationships.py b/cognite/client/_api/relationships.py new file mode 100644 index 0000000000..7f919254da --- /dev/null +++ b/cognite/client/_api/relationships.py @@ -0,0 +1,171 @@ +from typing import * + +from cognite.client import utils +from cognite.client._api_client import APIClient +from cognite.client.data_classes import Relationship, RelationshipList + + +class RelationshipsAPI(APIClient): + _RESOURCE_PATH = "/relationships" + _LIST_CLASS = RelationshipList + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._CREATE_LIMIT = 1000 + + def __call__( + self, chunk_size: int = None, limit: int = None + ) -> Generator[Union[Relationship, RelationshipList], None, None]: + """Iterate over relationships + + Fetches relationships as they are iterated over, so you keep a limited number of relationships in memory. + + Args: + chunk_size (int, optional): Number of relationships to return in each chunk. Defaults to yielding one relationship a time. + limit (int, optional): Maximum number of relationships to return. Defaults to 100. Set to -1, float("inf") or None + to return all items. + + Yields: + Union[Relationship, RelationshipList]: yields Relationship one by one if chunk is not specified, else RelationshipList objects. + """ + return self._list_generator(method="POST", chunk_size=chunk_size, limit=limit) + + def __iter__(self) -> Generator[Relationship, None, None]: + """Iterate over relationships + + Fetches relationships as they are iterated over, so you keep a limited number of relationships in memory. + + Yields: + Relationship: yields Relationships one by one. + """ + return self.__call__() + + def retrieve(self, external_id: str) -> Optional[Relationship]: + """Retrieve a single relationship by external id. + + Args: + external_id (str): External ID + + Returns: + Optional[Relationship]: Requested relationship or None if it does not exist. + + Examples: + + Get relationship by external id:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> res = c.relationships.retrieve(external_id="1") + """ + return self._retrieve_multiple(external_ids=external_id, wrap_ids=True) + + def retrieve_multiple(self, external_ids: List[str]) -> RelationshipList: + """Retrieve multiple relationships by external id. + + Args: + external_ids (List[str]): External IDs + + Returns: + RelationshipList: The requested relationships. + + Examples: + + Get relationships by external id:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> res = c.relationships.retrieve_multiple(external_ids=["abc", "def"]) + """ + utils._auxiliary.assert_type(external_ids, "external_id", [List], allow_none=False) + return self._retrieve_multiple(external_ids=external_ids, wrap_ids=True) + + def list(self, limit: int = 25) -> RelationshipList: + """List relationships + + Args: + limit (int, optional): Maximum number of relationships to return. Defaults to 100. Set to -1, float("inf") or None + to return all items. + + Returns: + RelationshipList: List of requested relationships + + Examples: + + List relationships:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> relationship_list = c.relationships.list(limit=5) + + Iterate over relationships:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> for relationship in c.relationships: + ... relationship # do something with the relationship + + Iterate over chunks of relationships to reduce memory load:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> for relationship_list in c.relationships(chunk_size=2500): + ... relationship_list # do something with the relationships + """ + return self._list(method="GET", limit=limit, filter=None) + + def create(self, relationship: Union[Relationship, List[Relationship]]) -> Union[Relationship, RelationshipList]: + """Create one or more relationships. + + Args: + relationship (Union[Relationship, List[Relationship]]): Relationship or list of relationships to create. + Note: the source and target field in the Relationship(s) can be of the form shown below, or objects of type Asset, TimeSeries, FileMetadata, Event + + Returns: + Union[Relationship, RelationshipList]: Created relationship(s) + + Examples: + + Create a new relationship:: + + >>> from cognite.client.experimental import CogniteClient + >>> from cognite.client.data_classes import Relationship + >>> c = CogniteClient() + >>> rel = Relationship(external_id="rel",source={"resource":"TimeSeries", "resourceId": "ts"},target={"resource":"Asset", "resourceId": "a"},relationship_type="belongsTo",confidence=0.9,data_set="ds_name") + >>> res = c.relationships.create(rel) + + Create a new relationship:: + + >>> from cognite.client.experimental import CogniteClient + >>> from cognite.client.data_classes import Relationship + >>> c = CogniteClient() + >>> assets = c.assets.retrieve_multiple(id=[1,2,3]) + >>> flowrel1 = Relationship(external_id="flow_1",source=assets[0],target=assets[1] ,relationship_type="flowsTo",confidence=0.1,data_set="ds_flow") + >>> flowrel2 = Relationship(external_id="flow_2",source=assets[1],target=assets[2] ,relationship_type="flowsTo",confidence=0.1,data_set="ds_flow") + >>> res = c.relationships.create([flowrel1,flowrel2]) + """ + utils._auxiliary.assert_type(relationship, "relationship", [Relationship, list]) + if isinstance(relationship, list): + relationship = [r._copy_resolve_targets() for r in relationship] + else: + relationship = relationship._copy_resolve_targets() + + return self._create_multiple(items=relationship) + + def delete(self, external_id: Union[str, List[str]]) -> None: + """Delete one or more relationships + + Args: + external_id (Union[str, List[str]]): External ID or list of external ids + + Returns: + None + + Examples: + + Delete relationships by external id:: + + >>> from cognite.client.experimental import CogniteClient + >>> c = CogniteClient() + >>> c.relationships.delete(external_id=["a","b"]) + """ + self._delete_multiple(external_ids=external_id, wrap_ids=True) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index a871ad5a1e..303419b804 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -16,6 +16,7 @@ ) from cognite.client.data_classes.login import LoginStatus from cognite.client.data_classes.raw import Database, DatabaseList, Row, RowList, Table, TableList +from cognite.client.data_classes.relationships import Relationship, RelationshipList from cognite.client.data_classes.sequences import Sequence, SequenceData, SequenceFilter, SequenceList, SequenceUpdate from cognite.client.data_classes.three_d import ( ThreeDAssetMapping, diff --git a/cognite/client/data_classes/relationships.py b/cognite/client/data_classes/relationships.py new file mode 100644 index 0000000000..366186edbb --- /dev/null +++ b/cognite/client/data_classes/relationships.py @@ -0,0 +1,79 @@ +import copy +from typing import * + +from cognite.client.data_classes._base import * + + +# GenClass: relationshipResponse, relationship +class Relationship(CogniteResource): + """No description. + + Args: + source (Dict[str, Any]): Reference by external id to the source of the relationship. Since it is a reference by external id, the targeted resource may or may not exist in CDF. If resource is `threeD` or `threeDRevision` the `resourceId` is a set of internal ids concatenated by a colons. Otherwise, the resourceId follows the formatting rules as described in `resourceId`. If resource id of type `threeD`, the externalId must follow the pattern `::`. If resource id of type `threeDRevision`, the externalId must follow the pattern `:`. The values ``, `` and `` are the corresponding internal ids to identify the referenced resource uniquely. + target (Dict[str, Any]): Reference by external id to the target of the relationship. Since it is a reference by external id, the targeted resource may or may not exist in CDF. If resource is `threeD` or `threeDRevision` the `resourceId` is a set of internal ids concatenated by a colons. Otherwise, the resourceId follows the formatting rules as described in `resourceId`. If resource id of type `threeD`, the externalId must follow the pattern `::`. If resource id of type `threeDRevision`, the externalId must follow the pattern `:`. The values ``, `` and `` are the corresponding internal ids to identify the referenced resource uniquely. + start_time (float): Time when this relationship was established in milliseconds since Jan 1, 1970. + end_time (float): Time when this relationship was ceased to exist in milliseconds since Jan 1, 1970. + confidence (float): Confidence value of the existence of this relationship. Humans should enter 1.0 usually, generated relationships should provide a realistic score on the likelihood of the existence of the relationship. Generated relationships should never have the a confidence score of 1.0. + data_set (str): String describing the source system storing or generating the relationship. + external_id (str): Disallowing leading and trailing whitespaces. Case sensitive. The external Id must be unique within the project. + relationship_type (str): Type of the relationship in order to distinguish between different relationships. E.g. a flow through a pipe can be naturally represented by a `flowsTo`-relationship. On the other hand an alternative asset hierarchy can be represented with the `isParentOf`-relationship. + created_time (float): Time when this relationship was created in CDF in milliseconds since Jan 1, 1970. + last_updated_time (float): Time when this relationship was last updated in CDF in milliseconds since Jan 1, 1970. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + source: Dict[str, Any] = None, + target: Dict[str, Any] = None, + start_time: float = None, + end_time: float = None, + confidence: float = None, + data_set: str = None, + external_id: str = None, + relationship_type: str = None, + created_time: float = None, + last_updated_time: float = None, + cognite_client=None, + ): + self.source = source + self.target = target + self.start_time = start_time + self.end_time = end_time + self.confidence = confidence + self.data_set = data_set + self.external_id = external_id + self.relationship_type = relationship_type + self.created_time = created_time + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + # GenStop + + def _copy_resolve_targets(self): + rel = copy.copy(self) + rel.source = self._resolve_target(rel.source) + rel.target = self._resolve_target(rel.target) + return rel + + @staticmethod + def _resolve_target(target): + if isinstance(target, dict): + return target + + from cognite.client.data_classes import Asset, Event, FileMetadata, TimeSeries + + _TARGET_TYPES = {Asset: "Asset", TimeSeries: "TimeSeries", FileMetadata: "File", Event: "Event"} + typestr = _TARGET_TYPES.get(target.__class__) + if typestr: + return {"resource": typestr, "resourceId": target.external_id} + raise ValueError("Invalid source or target '{}' of type {} in relationship".format(target, target.__class__)) + + +class RelationshipUpdate(CogniteUpdate): + pass + + +class RelationshipList(CogniteResourceList): + _RESOURCE = Relationship + _UPDATE = RelationshipUpdate diff --git a/cognite/client/experimental.py b/cognite/client/experimental.py index 5445739967..ab27f81ecb 100644 --- a/cognite/client/experimental.py +++ b/cognite/client/experimental.py @@ -1,4 +1,5 @@ from cognite.client._api.model_hosting import ModelHostingAPI +from cognite.client._api.relationships import RelationshipsAPI from cognite.client._api.sequences import SequencesAPI from cognite.client._cognite_client import CogniteClient as Client @@ -8,3 +9,4 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.sequences = SequencesAPI(self._config, api_version=self._API_VERSION, cognite_client=self) self.model_hosting = ModelHostingAPI(self._config, api_version="0.6", cognite_client=self) + self.relationships = RelationshipsAPI(self._config, api_version="playground", cognite_client=self) diff --git a/docs/source/cognite.rst b/docs/source/cognite.rst index 111cf8efc0..1916f58c69 100644 --- a/docs/source/cognite.rst +++ b/docs/source/cognite.rst @@ -1026,3 +1026,35 @@ Data classes .. automodule:: cognite.client.data_classes.sequences :members: :show-inheritance: + + +Relationships +------------- +.. WARNING:: + The relationships API is experimental and subject to breaking changes. It should not be used in production code. + +Retrieve a relationship by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.relationships.RelationshipsAPI.retrieve + +Retrieve multiple relationships by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.relationships.RelationshipsAPI.retrieve_multiple + +List relationships +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.relationships.RelationshipsAPI.list + +Create a relationship +^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.relationships.RelationshipsAPI.create + +Delete relationships +^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.relationships.RelationshipsAPI.delete + +Data classes +^^^^^^^^^^^^ +.. automodule:: cognite.client.data_classes.relationships + :members: + :show-inheritance: \ No newline at end of file diff --git a/generate_from_spec.py b/generate_from_spec.py index 75a639edb3..546b1b7025 100644 --- a/generate_from_spec.py +++ b/generate_from_spec.py @@ -22,7 +22,7 @@ def main(spec_url, spec_path): for file in files: file_path = os.path.join(root, file) if file_path.endswith(".py"): - if "sequence" in file_path: + if "sequence" in file_path or "relationships" in file_path: print("* Generating playground code in {}".format(file_path)) codegen_playground.generate(file_path, file_path) else: diff --git a/openapi/generator.py b/openapi/generator.py index 70c1ac452a..de2d6b9311 100644 --- a/openapi/generator.py +++ b/openapi/generator.py @@ -84,9 +84,11 @@ def generate_constructor(self, schemas, indentation): @staticmethod def _get_schema_description(schema): + desc = schema.get("description") if "allOf" in schema: schema = schema["allOf"][0] - return schema.get("description", "No description.") + desc = desc or schema.get("description") + return (desc or "No description.").replace("\n", " ") @staticmethod def _get_schema_properties(schema): diff --git a/tests/conftest.py b/tests/conftest.py index 65be192760..712936b0be 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,7 @@ from cognite.client._api.iam import IAMAPI, APIKeysAPI, GroupsAPI, SecurityCategoriesAPI, ServiceAccountsAPI from cognite.client._api.login import LoginAPI from cognite.client._api.raw import RawAPI, RawDatabasesAPI, RawRowsAPI, RawTablesAPI +from cognite.client._api.relationships import RelationshipsAPI from cognite.client._api.sequences import SequencesAPI, SequencesDataAPI from cognite.client._api.three_d import ( ThreeDAPI, @@ -70,8 +71,10 @@ def mock_cognite_client(): def mock_cognite_experimental_client(mock_cognite_client): with mock.patch("cognite.client.experimental.CogniteClient") as client_mock: cog_client_mock = mock.MagicMock(spec=CogniteClient) + cog_client_mock.assets = mock.MagicMock(spec=AssetsAPI) cog_client_mock.sequences = mock.MagicMock(spec=SequencesAPI) cog_client_mock.sequences.data = mock.MagicMock(spec=SequencesDataAPI) + cog_client_mock.relationships = mock.MagicMock(spec=RelationshipsAPI) client_mock.return_value = cog_client_mock yield diff --git a/tests/tests_unit/test_api/test_relationships.py b/tests/tests_unit/test_api/test_relationships.py new file mode 100644 index 0000000000..938a723ce4 --- /dev/null +++ b/tests/tests_unit/test_api/test_relationships.py @@ -0,0 +1,181 @@ +import gzip +import json +import math +import os +import re +from unittest import mock + +import pytest + +from cognite.client.data_classes import ( + Asset, + Event, + FileMetadata, + Relationship, + RelationshipList, + Sequence, + ThreeDModel, + ThreeDModelRevision, + TimeSeries, +) +from cognite.client.experimental import CogniteClient +from tests.utils import jsgz_load + +COGNITE_CLIENT = CogniteClient() +REL_API = COGNITE_CLIENT.relationships + + +@pytest.fixture +def mock_rel_response(rsps): + response_body = { + "items": [ + { + "externalId": "rel-123", + "createdTime": 1565965333132, + "lastUpdatedTime": 1565965333132, + "confidence": 0.99, + "dataSet": "testSource", + "relationshipType": "flowsTo", + "source": {"resourceId": "asset1", "resource": "Asset"}, + "target": {"resourceId": "asset2", "resource": "Asset"}, + } + ] + } + url_pattern = re.compile( + re.escape(REL_API._get_base_url_with_base_path()) + + r"/relationships(?:/byids|/update|/delete|/list|/search|$|\?.+)" + ) + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_rel_empty(rsps): + response_body = {"items": []} + url_pattern = re.compile( + re.escape(REL_API._get_base_url_with_base_path()) + + r"/relationships(?:/byids|/update|/delete|/list|/search|$|\?.+)" + ) + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestRelationships: + def test_retrieve_single(self, mock_rel_response): + res = REL_API.retrieve(external_id="a") + assert isinstance(res, Relationship) + assert mock_rel_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + assert {"items": [{"externalId": "a"}]} == jsgz_load(mock_rel_response.calls[0].request.body) + + def test_retrieve_multiple(self, mock_rel_response): + res = REL_API.retrieve_multiple(external_ids=["a"]) + assert isinstance(res, RelationshipList) + assert mock_rel_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"items": [{"externalId": "a"}]} == jsgz_load(mock_rel_response.calls[0].request.body) + + def test_list(self, mock_rel_response): + res = REL_API.list() + assert mock_rel_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create_single(self, mock_rel_response): + res = REL_API.create( + Relationship( + external_id="1", + confidence=0.5, + relationship_type="flowsTo", + source={"resourceId": "aaa", "resource": "Asset"}, + target={"resourceId": "bbb", "resource": "Asset"}, + ) + ) + assert isinstance(res, Relationship) + assert mock_rel_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_single_types(self, mock_rel_response): + types = [Asset, TimeSeries, FileMetadata, Event] + for cls in types: + print(cls) + test = cls(external_id="test") + res = REL_API.create( + Relationship( + external_id="1", + confidence=0.5, + relationship_type="belongsTo", + source=test, + target={"resourceId": "bbb", "resource": "Asset"}, + ) + ) + assert isinstance(res, Relationship) + res = REL_API.create( + Relationship( + external_id="1", + confidence=0.5, + relationship_type="belongsTo", + source={"resourceId": "bbb", "resource": "Asset"}, + target=test, + ) + ) + assert isinstance(res, Relationship) + res = REL_API.create( + Relationship(external_id="1", confidence=0.5, relationship_type="belongsTo", source=test, target=test) + ) + assert isinstance(res, Relationship) + + for call in mock_rel_response.calls: + it = json.loads(gzip.decompress(call.request.body).decode("utf-8"))["items"][0] + assert isinstance(it["source"], dict) + assert isinstance(it["target"], dict) + + def test_create_wrong_type(self, mock_rel_response): + with pytest.raises(ValueError): + res = REL_API.create( + Relationship( + external_id="1", + confidence=0.5, + relationship_type="belongsTo", + source=Sequence(external_id="a'"), + target={"resourceId": "bbb", "resource": "Asset"}, + ) + ) + + def test_create_multiple(self, mock_rel_response): + rel1 = Relationship( + external_id="new1", + confidence=0.5, + relationship_type="flowsTo", + source={"resourceId": "aaa", "resource": "Asset"}, + target={"resourceId": "bbb", "resource": "Asset"}, + ) + rel2 = Relationship( + external_id="new2", + confidence=0.1, + relationship_type="flowsTo", + source={"resourceId": "aaa", "resource": "Asset"}, + target={"resourceId": "bbb", "resource": "Asset"}, + ) + res = REL_API.create([rel1, rel2]) + assert isinstance(res, RelationshipList) + assert mock_rel_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_iter_single(self, mock_rel_response): + for rel in REL_API: + assert mock_rel_response.calls[0].response.json()["items"][0] == rel.dump(camel_case=True) + + def test_iter_chunk(self, mock_rel_response): + for rel in REL_API(chunk_size=1): + assert mock_rel_response.calls[0].response.json()["items"] == rel.dump(camel_case=True) + + def test_delete_single(self, mock_rel_response): + res = REL_API.delete(external_id="a") + assert {"items": [{"externalId": "a"}]} == jsgz_load(mock_rel_response.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_rel_response): + res = REL_API.delete(external_id=["a"]) + assert {"items": [{"externalId": "a"}]} == jsgz_load(mock_rel_response.calls[0].request.body) + assert res is None diff --git a/tests/tests_unit/test_docstring_examples.py b/tests/tests_unit/test_docstring_examples.py index 2cab30b9a7..cb06a24252 100644 --- a/tests/tests_unit/test_docstring_examples.py +++ b/tests/tests_unit/test_docstring_examples.py @@ -3,7 +3,19 @@ import pytest -from cognite.client._api import assets, datapoints, events, files, iam, login, raw, sequences, three_d, time_series +from cognite.client._api import ( + assets, + datapoints, + events, + files, + iam, + login, + raw, + relationships, + sequences, + three_d, + time_series, +) # this fixes the issue with 'got MagicMock but expected Nothing in docstrings' doctest.OutputChecker.__check_output = doctest.OutputChecker.check_output @@ -53,3 +65,6 @@ def test_iam(self): class TestDocstringExamplesExperimental: def test_sequences(self): run_docstring_tests(sequences) + + def test_relationships(self): + run_docstring_tests(relationships)