diff --git a/.gitignore b/.gitignore index e4fdda3e36..2e4d924fee 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ wheels/ # Other test.py +do_release.py .env # Coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index 5978f99472..59dee1c3d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,30 @@ Changes are grouped as follows - `Security` in case of vulnerabilities. ## [Unreleased] +- Concurrent reads for all resource types using `/cursors` endpoints + +## [1.0.0] ### Added - Support for all endpoints in Cognite API +- Generator with hidden cursor for all resource types +- Concurrent writes for all resources +- Distribution of "core" sdk which does not depend on pandas and numpy +- Typehints for all methods +- Support for posting an entire asset hierarchy, resolving ref_id/parent_ref_id automatically ### Removed - `experimental` client in order to ensure sdk stability. ### Changed -- Rename methods so they reflect what the method does instead of what http method is used +- Renamed methods so they reflect what the method does instead of what http method is used +- Updated documentation with automatically tested examples +- Renamed `stable` namespace to `api` +- Rewrote logic for concurrent reads of datapoints +- Renamed CogniteClient parameter `num_of_workers` to `max_workers` +- Empty artifacts folder now raises exception when building source package in `model_hosting`. + +## Fixed +- Bug causing `create_schedule` to yield 400 if description is not set ## [0.13.3] - 2019-03-25 ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c631f6a793..9a33e14dd6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,8 +7,8 @@ $ cd cognite-sdk-python ``` Install dependencies and initialize a shell within the virtual environment. ```bash -$ pipenv install -d $ pipenv shell +$ pipenv sync -d ``` Install pre-commit hooks ```bash @@ -19,14 +19,18 @@ Set up tests for all new functionality. Running the tests will require setting t Initiate unit tests by running the following command from the root directory: -`$ pytest` +`$ pytest tests/tests_unit` + +If you have an appropriate API key, you can run the integratino tests like this: + +`$ pytest tests/tests_integration` If you want to generate code coverage reports run: ``` -pytest --cov-report html \ - --cov-report xml \ - --cov cognite +pytest tests/tests_unit --cov-report html \ + --cov-report xml \ + --cov cognite ``` Open `htmlcov/index.html` in the browser to navigate through the report. diff --git a/Jenkinsfile b/Jenkinsfile index c49c0fddfb..a2e0c6a5d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,14 @@ podTemplate( resourceLimitCpu: '1000m', resourceLimitMemory: '800Mi', ttyEnabled: true), + containerTemplate(name: 'node', + image: 'node:slim', + command: '/bin/cat -', + resourceRequestCpu: '300m', + resourceRequestMemory: '300Mi', + resourceLimitCpu: '300m', + resourceLimitMemory: '300Mi', + ttyEnabled: true), ], volumes: [ secretVolume(secretName: 'jenkins-docker-builder', mountPath: '/jenkins-docker-builder', readOnly: true), @@ -24,8 +32,11 @@ podTemplate( configMapVolume(configMapName: 'codecov-script-configmap', mountPath: '/codecov-script'), ], envVars: [ - secretEnvVar(key: 'COGNITE_API_KEY', secretName: 'ml-test-api-key', secretKey: 'testkey.txt'), + secretEnvVar(key: 'COGNITE_API_KEY', secretName: 'cognite-sdk-python', secretKey: 'integration-test-api-key'), secretEnvVar(key: 'CODECOV_TOKEN', secretName: 'codecov-token-cognite-sdk-python', secretKey: 'token.txt'), + envVar(key: 'COGNITE_BASE_URL', value: "https://greenfield.cognitedata.com"), + envVar(key: 'COGNITE_CLIENT_NAME', value: "python-sdk-integration-tests"), + envVar(key: 'CI', value: '1'), // /codecov-script/upload-report.sh relies on the following // Jenkins and Github environment variables. envVar(key: 'BRANCH_NAME', value: env.BRANCH_NAME), @@ -41,11 +52,24 @@ podTemplate( gitCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() } } + container('node'){ + stage('Download and dereference OpenAPI Spec'){ + sh('npm install -g swagger-cli') + sh('curl https://storage.googleapis.com/cognitedata-api-docs/dist/v1.json --output spec.json') + sh('swagger-cli bundle -r spec.json -o deref-spec.json') + } + } container('python') { stage('Install pipenv') { sh("pip3 install pipenv") } - stage('Install dependencies') { + stage('Install core dependencies') { + sh("pipenv run pip install -r core-requirements.txt") + } + stage('Test core') { + sh("pipenv run pytest tests/tests_unit -m 'not dsl' --test-deps-only-core") + } + stage('Install all dependencies') { sh("pipenv sync --dev") } stage('Check code') { @@ -58,9 +82,12 @@ podTemplate( sh("pipenv run sphinx-build -W -b html ./source ./build") } } - stage('Test and coverage report') { + stage('Test OpenAPI Generator'){ + sh('pipenv run pytest openapi/tests') + } + stage('Test Client') { sh("pyenv local 3.5.0 3.6.6 3.7.2") - sh("pipenv run tox") + sh("pipenv run tox -p auto") junit(allowEmptyResults: true, testResults: '**/test-report.xml') summarizeTestResults() } @@ -71,14 +98,15 @@ podTemplate( stage('Build') { sh("python3 setup.py sdist") sh("python3 setup.py bdist_wheel") + sh("python3 setup-core.py sdist") + sh("python3 setup-core.py bdist_wheel") } - def pipVersion = sh(returnStdout: true, script: 'pipenv run yolk -V cognite-sdk | sort -n | tail -1 | cut -d\\ -f 2').trim() def currentVersion = sh(returnStdout: true, script: 'sed -n -e "/^__version__/p" cognite/client/__init__.py | cut -d\\" -f2').trim() - println("This version: " + currentVersion) - println("Latest pip version: " + pipVersion) - if (env.BRANCH_NAME == 'master' && currentVersion != pipVersion) { + def versionExists = sh(returnStdout: true, script: 'pipenv run python3 cognite/client/utils/_version_checker.py -p cognite-sdk -v ' + currentVersion) + println("Version Exists: " + versionExists) + if (env.BRANCH_NAME == 'master' && versionExists == 'no') { stage('Release') { sh("pipenv run twine upload --config-file /pypi/.pypirc dist/*") } diff --git a/Pipfile b/Pipfile index 68960cdc5e..be8af884b5 100644 --- a/Pipfile +++ b/Pipfile @@ -4,25 +4,24 @@ verify_ssl = true name = "pypi" [packages] -sphinx = "*" pandas = "*" requests = "*" -protobuf = "*" -sphinx-rtd-theme = "*" -cognite-logger = "==0.4.*" -pytest-mock = "*" +matplotlib = "*" [dev-packages] +sphinx = "*" +sphinx-rtd-theme = "*" pytest-cov = "*" twine = "*" -"yolk3k" = "*" pytest = "*" -black = "==18.6b4" +black = "==19.3b0" strip-hints = "*" pre-commit = "*" isort = "*" tox = "*" tox-pyenv = "*" +responses = "*" +pytest-mock = "*" [pipenv] allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock index 5e25495510..420e7a3ccc 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "37680e235844769395a200637429e4b1eb300d831b20a9979db07406700143da" + "sha256": "c3ea2725ff7cc9b85b535b9fc6d7903bfd05880342b2dbeedfb1c8b69e75477e" }, "pipfile-spec": 6, "requires": {}, @@ -14,34 +14,6 @@ ] }, "default": { - "alabaster": { - "hashes": [ - "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", - "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" - ], - "version": "==0.7.12" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, - "attrs": { - "hashes": [ - "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", - "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" - ], - "version": "==19.1.0" - }, - "babel": { - "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" - ], - "version": "==2.6.0" - }, "certifi": { "hashes": [ "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", @@ -56,20 +28,12 @@ ], "version": "==3.0.4" }, - "cognite-logger": { - "hashes": [ - "sha256:d81e13887b598afe9992cf233cad74629df4a6d287b5cff04b5f2630909ff9fb" - ], - "index": "pypi", - "version": "==0.4.0" - }, - "docutils": { + "cycler": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:1d8a5ae1ff6c5cf9b93e8811e581232ad8920aeec647c37316ceac982b08cb2d", + "sha256:cd7b2d1018258d7247a71425e9f26463dfb444d411c39569972f4ce586b0c9d8" ], - "version": "==0.14" + "version": "==0.10.0" }, "idna": { "hashes": [ @@ -78,166 +42,107 @@ ], "version": "==2.8" }, - "imagesize": { - "hashes": [ - "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", - "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" + "kiwisolver": { + "hashes": [ + "sha256:05b5b061e09f60f56244adc885c4a7867da25ca387376b02c1efc29cc16bcd0f", + "sha256:26f4fbd6f5e1dabff70a9ba0d2c4bd30761086454aa30dddc5b52764ee4852b7", + "sha256:3b2378ad387f49cbb328205bda569b9f87288d6bc1bf4cd683c34523a2341efe", + "sha256:400599c0fe58d21522cae0e8b22318e09d9729451b17ee61ba8e1e7c0346565c", + "sha256:47b8cb81a7d18dbaf4fed6a61c3cecdb5adec7b4ac292bddb0d016d57e8507d5", + "sha256:53eaed412477c836e1b9522c19858a8557d6e595077830146182225613b11a75", + "sha256:58e626e1f7dfbb620d08d457325a4cdac65d1809680009f46bf41eaf74ad0187", + "sha256:5a52e1b006bfa5be04fe4debbcdd2688432a9af4b207a3f429c74ad625022641", + "sha256:5c7ca4e449ac9f99b3b9d4693debb1d6d237d1542dd6a56b3305fe8a9620f883", + "sha256:682e54f0ce8f45981878756d7203fd01e188cc6c8b2c5e2cf03675390b4534d5", + "sha256:79bfb2f0bd7cbf9ea256612c9523367e5ec51d7cd616ae20ca2c90f575d839a2", + "sha256:7f4dd50874177d2bb060d74769210f3bce1af87a8c7cf5b37d032ebf94f0aca3", + "sha256:8944a16020c07b682df861207b7e0efcd2f46c7488619cb55f65882279119389", + "sha256:8aa7009437640beb2768bfd06da049bad0df85f47ff18426261acecd1cf00897", + "sha256:939f36f21a8c571686eb491acfffa9c7f1ac345087281b412d63ea39ca14ec4a", + "sha256:9733b7f64bd9f807832d673355f79703f81f0b3e52bfce420fc00d8cb28c6a6c", + "sha256:a02f6c3e229d0b7220bd74600e9351e18bc0c361b05f29adae0d10599ae0e326", + "sha256:a0c0a9f06872330d0dd31b45607197caab3c22777600e88031bfe66799e70bb0", + "sha256:acc4df99308111585121db217681f1ce0eecb48d3a828a2f9bbf9773f4937e9e", + "sha256:b64916959e4ae0ac78af7c3e8cef4becee0c0e9694ad477b4c6b3a536de6a544", + "sha256:d3fcf0819dc3fea58be1fd1ca390851bdb719a549850e708ed858503ff25d995", + "sha256:d52e3b1868a4e8fd18b5cb15055c76820df514e26aa84cc02f593d99fef6707f", + "sha256:db1a5d3cc4ae943d674718d6c47d2d82488ddd94b93b9e12d24aabdbfe48caee", + "sha256:e3a21a720791712ed721c7b95d433e036134de6f18c77dbe96119eaf7aa08004", + "sha256:e8bf074363ce2babeb4764d94f8e65efd22e6a7c74860a4f05a6947afc020ff2", + "sha256:f16814a4a96dc04bf1da7d53ee8d5b1d6decfc1a92a63349bb15d37b6a263dd9", + "sha256:f2b22153870ca5cf2ab9c940d7bc38e8e9089fa0f7e5856ea195e1cf4ff43d5a", + "sha256:f790f8b3dff3d53453de6a7b7ddd173d2e020fb160baff578d578065b108a05f" ], "version": "==1.1.0" }, - "jinja2": { - "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" - ], - "version": "==2.10.1" - }, - "markupsafe": { + "matplotlib": { "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + "sha256:056fc93273d427fed5600bf6125adcdd600e242c4edf72723dd766fe26b333b1", + "sha256:375b694ad0426314132ed6d324d1a5c05d121f906b08b43d314c2d5a7cd75332", + "sha256:4ff5ea3530cd5f847128a0cdf58f5a4880491ac412a8160870d647e7fb849edd", + "sha256:570beedeec485a8ec2ce7940c213be005265c8697a347e299d2ccc8c3c2e1fe0", + "sha256:7f11ae7cb767d856b792fdf02c8e5c339a1327b7ada19c5b36b64f5a72562f7a", + "sha256:93c54e96e88da1700127fea79fcef2b3e340e3342758655b93db8c9b93d4d1e6", + "sha256:ad6f0a8e0e31e1e5342029fa0bc44aa3c54a21eac9979559dea7da7f1799299d", + "sha256:b9a7cea9ab675c56140339ff34fd5f71c1d1afef7e8522b5f14905b1555febbe", + "sha256:ca69dd0ebbb31f2148fce5fbf8ba1d3e73c384d53d74b92445c3093ec4f1f7de" ], - "version": "==1.1.1" - }, - "more-itertools": { - "hashes": [ - "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", - "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" - ], - "markers": "python_version > '2.7'", - "version": "==7.0.0" + "index": "pypi", + "version": "==3.1.0rc1" }, "numpy": { "hashes": [ - "sha256:1980f8d84548d74921685f68096911585fee393975f53797614b34d4f409b6da", - "sha256:22752cd809272671b273bb86df0f505f505a12368a3a5fc0aa811c7ece4dfd5c", - "sha256:23cc40313036cffd5d1873ef3ce2e949bdee0646c5d6f375bf7ee4f368db2511", - "sha256:2b0b118ff547fecabc247a2668f48f48b3b1f7d63676ebc5be7352a5fd9e85a5", - "sha256:3a0bd1edf64f6a911427b608a894111f9fcdb25284f724016f34a84c9a3a6ea9", - "sha256:3f25f6c7b0d000017e5ac55977a3999b0b1a74491eacb3c1aa716f0e01f6dcd1", - "sha256:4061c79ac2230594a7419151028e808239450e676c39e58302ad296232e3c2e8", - "sha256:560ceaa24f971ab37dede7ba030fc5d8fa173305d94365f814d9523ffd5d5916", - "sha256:62be044cd58da2a947b7e7b2252a10b42920df9520fc3d39f5c4c70d5460b8ba", - "sha256:6c692e3879dde0b67a9dc78f9bfb6f61c666b4562fd8619632d7043fb5b691b0", - "sha256:6f65e37b5a331df950ef6ff03bd4136b3c0bbcf44d4b8e99135d68a537711b5a", - "sha256:7a78cc4ddb253a55971115f8320a7ce28fd23a065fc33166d601f51760eecfa9", - "sha256:80a41edf64a3626e729a62df7dd278474fc1726836552b67a8c6396fd7e86760", - "sha256:893f4d75255f25a7b8516feb5766c6b63c54780323b9bd4bc51cdd7efc943c73", - "sha256:972ea92f9c1b54cc1c1a3d8508e326c0114aaf0f34996772a30f3f52b73b942f", - "sha256:9f1d4865436f794accdabadc57a8395bd3faa755449b4f65b88b7df65ae05f89", - "sha256:9f4cd7832b35e736b739be03b55875706c8c3e5fe334a06210f1a61e5c2c8ca5", - "sha256:adab43bf657488300d3aeeb8030d7f024fcc86e3a9b8848741ea2ea903e56610", - "sha256:bd2834d496ba9b1bdda3a6cf3de4dc0d4a0e7be306335940402ec95132ad063d", - "sha256:d20c0360940f30003a23c0adae2fe50a0a04f3e48dc05c298493b51fd6280197", - "sha256:d3b3ed87061d2314ff3659bb73896e622252da52558f2380f12c421fbdee3d89", - "sha256:dc235bf29a406dfda5790d01b998a1c01d7d37f449128c0b1b7d1c89a84fae8b", - "sha256:fb3c83554f39f48f3fa3123b9c24aecf681b1c289f9334f8215c1d3c8e2f6e5b" - ], - "version": "==1.16.2" - }, - "packaging": { - "hashes": [ - "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", - "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" - ], - "version": "==19.0" + "sha256:0e2eed77804b2a6a88741f8fcac02c5499bba3953ec9c71e8b217fad4912c56c", + "sha256:1c666f04553ef70fda54adf097dbae7080645435fc273e2397f26bbf1d127bbb", + "sha256:1f46532afa7b2903bfb1b79becca2954c0a04389d19e03dc73f06b039048ac40", + "sha256:315fa1b1dfc16ae0f03f8fd1c55f23fd15368710f641d570236f3d78af55e340", + "sha256:3d5fcea4f5ed40c3280791d54da3ad2ecf896f4c87c877b113576b8280c59441", + "sha256:48241759b99d60aba63b0e590332c600fc4b46ad597c9b0a53f350b871ef0634", + "sha256:4b4f2924b36d857cf302aec369caac61e43500c17eeef0d7baacad1084c0ee84", + "sha256:54fe3b7ed9e7eb928bbc4318f954d133851865f062fa4bbb02ef8940bc67b5d2", + "sha256:5a8f021c70e6206c317974c93eaaf9bc2b56295b6b1cacccf88846e44a1f33fc", + "sha256:754a6be26d938e6ca91942804eb209307b73f806a1721176278a6038869a1686", + "sha256:771147e654e8b95eea1293174a94f34e2e77d5729ad44aefb62fbf8a79747a15", + "sha256:78a6f89da87eeb48014ec652a65c4ffde370c036d780a995edaeb121d3625621", + "sha256:7fde5c2a3a682a9e101e61d97696687ebdba47637611378b4127fe7e47fdf2bf", + "sha256:80d99399c97f646e873dd8ce87c38cfdbb668956bbc39bc1e6cac4b515bba2a0", + "sha256:88a72c1e45a0ae24d1f249a529d9f71fe82e6fa6a3fd61414b829396ec585900", + "sha256:a4f4460877a16ac73302a9c077ca545498d9fe64e6a81398d8e1a67e4695e3df", + "sha256:a61255a765b3ac73ee4b110b28fccfbf758c985677f526c2b4b39c48cc4b509d", + "sha256:ab4896a8c910b9a04c0142871d8800c76c8a2e5ff44763513e1dd9d9631ce897", + "sha256:abbd6b1c2ef6199f4b7ca9f818eb6b31f17b73a6110aadc4e4298c3f00fab24e", + "sha256:b16d88da290334e33ea992c56492326ea3b06233a00a1855414360b77ca72f26", + "sha256:b78a1defedb0e8f6ae1eb55fa6ac74ab42acc4569c3a2eacc2a407ee5d42ebcb", + "sha256:cfef82c43b8b29ca436560d51b2251d5117818a8d1fb74a8384a83c096745dad", + "sha256:d160e57731fcdec2beda807ebcabf39823c47e9409485b5a3a1db3a8c6ce763e" + ], + "version": "==1.16.3" }, "pandas": { "hashes": [ - "sha256:02c830f951f3dc8c3164e2639a8961881390f7492f71a7835c2330f54539ad57", - "sha256:179015834c72a577486337394493cc2969feee9a04a2ea09f50c724e4b52ab42", - "sha256:3894960d43c64cfea5142ac783b101362f5008ee92e962392156a3f8d1558995", - "sha256:435821cb2501eabbcee7e83614bd710940dc0cf28b5afbc4bdb816c31cec71af", - "sha256:8294dea9aa1811f93558702856e3b68dd1dfd7e9dbc8e0865918a07ee0f21c2c", - "sha256:844e745ab27a9a01c86925fe776f9d2e09575e65f0bf8eba5090edddd655dffc", - "sha256:a08d49f5fa2a2243262fe5581cb89f6c0c7cc525b8d6411719ab9400a9dc4a82", - "sha256:a435c251246075337eb9fdc4160fd15c8a87cc0679d8d61fb5255d8d5a12f044", - "sha256:a799f03c0ec6d8687f425d7d6c075e8055a9a808f1ba87604d91f20507631d8d", - "sha256:aea72ce5b3a016b578cc05c04a2f68d9cafacf5d784b6fe832e66381cb62c719", - "sha256:c145e94c6da2af7eaf1fd827293ac1090a61a9b80150bebe99f8966a02378db9", - "sha256:c8a7b470c88c779301b73b23cabdbbd94b83b93040b2ccffa409e06df23831c0", - "sha256:c9e31b36abbd7b94c547d9047f13e1546e3ba967044cf4f9718575fcb7b81bb6", - "sha256:d960b7a03c33c328c723cfc2f8902a6291645f4efa0a5c1d4c5fa008cdc1ea77", - "sha256:da21fae4c173781b012217c9444f13c67449957a4d45184a9718268732c09564", - "sha256:db26c0fea0bd7d33c356da98bafd2c0dfb8f338e45e2824ff8f4f3e61b5c5f25", - "sha256:dc296c3f16ec620cfb4daf0f672e3c90f3920ece8261b2760cd0ebd9cd4daa55", - "sha256:e8da67cb2e9333ec30d53cfb96e27a4865d1648688e5471699070d35d8ab38cf", - "sha256:fb4f047a63f91f22aade4438aaf790400b96644e802daab4293e9b799802f93f", - "sha256:fef9939176cba0c2526ebeefffb8b9807543dc0954877b7226f751ec1294a869" + "sha256:071e42b89b57baa17031af8c6b6bbd2e9a5c68c595bc6bf9adabd7a9ed125d3b", + "sha256:17450e25ae69e2e6b303817bdf26b2cd57f69595d8550a77c308be0cd0fd58fa", + "sha256:17916d818592c9ec891cbef2e90f98cc85e0f1e89ed0924c9b5220dc3209c846", + "sha256:2538f099ab0e9f9c9d09bbcd94b47fd889bad06dc7ae96b1ed583f1dc1a7a822", + "sha256:366f30710172cb45a6b4f43b66c220653b1ea50303fbbd94e50571637ffb9167", + "sha256:42e5ad741a0d09232efbc7fc648226ed93306551772fc8aecc6dce9f0e676794", + "sha256:4e718e7f395ba5bfe8b6f6aaf2ff1c65a09bb77a36af6394621434e7cc813204", + "sha256:4f919f409c433577a501e023943e582c57355d50a724c589e78bc1d551a535a2", + "sha256:4fe0d7e6438212e839fc5010c78b822664f1a824c0d263fd858f44131d9166e2", + "sha256:5149a6db3e74f23dc3f5a216c2c9ae2e12920aa2d4a5b77e44e5b804a5f93248", + "sha256:627594338d6dd995cfc0bacd8e654cd9e1252d2a7c959449228df6740d737eb8", + "sha256:83c702615052f2a0a7fb1dd289726e29ec87a27272d775cb77affe749cca28f8", + "sha256:8c872f7fdf3018b7891e1e3e86c55b190e6c5cee70cab771e8f246c855001296", + "sha256:90f116086063934afd51e61a802a943826d2aac572b2f7d55caaac51c13db5b5", + "sha256:a3352bacac12e1fc646213b998bce586f965c9d431773d9e91db27c7c48a1f7d", + "sha256:bcdd06007cca02d51350f96debe51331dec429ac8f93930a43eb8fb5639e3eb5", + "sha256:c1bd07ebc15285535f61ddd8c0c75d0d6293e80e1ee6d9a8d73f3f36954342d0", + "sha256:c9a4b7c55115eb278c19aa14b34fcf5920c8fe7797a09b7b053ddd6195ea89b3", + "sha256:cc8fc0c7a8d5951dc738f1c1447f71c43734244453616f32b8aa0ef6013a5dfb", + "sha256:d7b460bc316064540ce0c41c1438c416a40746fd8a4fb2999668bf18f3c4acf1" ], "index": "pypi", - "version": "==0.24.1" - }, - "pluggy": { - "hashes": [ - "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", - "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" - ], - "version": "==0.9.0" - }, - "protobuf": { - "hashes": [ - "sha256:03666634d038e35d90155756914bc3a6316e8bcc0d300f3ee539e586889436b9", - "sha256:049d5900e442d4cc0fd2afd146786b429151e2b29adebed28e6376026ab0ee0b", - "sha256:0eb9e62a48cc818b1719b5035042310c7e4f57b01f5283b32998c68c2f1c6a7c", - "sha256:255d10c2c9059964f6ebb5c900a830fc8a089731dda94a5cc873f673193d208b", - "sha256:358cc59e4e02a15d3725f204f2eb5777fc10595e2d9a9c4c8d82292f49af6d41", - "sha256:41f1b737d5f97f1e2af23d16fac6c0b8572f9c7ea73054f1258ca57f4f97cb80", - "sha256:6a5129576a2cf925cd100e06ead5f9ae4c86db70a854fb91cedb8d680112734a", - "sha256:80722b0d56dcb7ca8f75f99d8dadd7c7efd0d2265714d68f871ed437c32d82b3", - "sha256:88a960e949ec356f7016d84f8262dcff2b842fca5355b4c1be759f5c103b19b3", - "sha256:97872686223f47d95e914881cb0ca46e1bc622562600043da9edddcb54f2fe1e", - "sha256:a1df9d22433ab44b7c7e0bd33817134832ae8a8f3d93d9b9719fc032c5b20e96", - "sha256:ad385fbb9754023d17be14dd5aa67efff07f43c5df7f93118aef3c20e635ea19", - "sha256:b2d5ee7ba5c03b735c02e6ae75fd4ff8c831133e7ca078f2963408dc7beac428", - "sha256:c8c07cd8635d45b28ec53ee695e5ac8b0f9d9a4ae488a8d8ee168fe8fc75ba43", - "sha256:d44ebc9838b183e8237e7507885d52e8d08c48fdc953fd4a7ee3e56cb9d20977", - "sha256:dff97b0ee9256f0afdfc9eaa430736cdcdc18899d9a666658f161afd137cf93d", - "sha256:e47d248d614c68e4b029442de212bdd4f6ae02ae36821de319ae90314ea2578c", - "sha256:e650b521b429fed3d525428b1401a40051097a5a92c30076c91f36b31717e087" - ], - "index": "pypi", - "version": "==3.7.0" - }, - "py": { - "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" - ], - "version": "==1.8.0" - }, - "pygments": { - "hashes": [ - "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", - "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" - ], - "version": "==2.3.1" + "version": "==0.24.2" }, "pyparsing": { "hashes": [ @@ -246,21 +151,6 @@ ], "version": "==2.4.0" }, - "pytest": { - "hashes": [ - "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", - "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" - ], - "version": "==4.4.1" - }, - "pytest-mock": { - "hashes": [ - "sha256:4d0d06d173eecf172703219a71dbd4ade0e13904e6bbce1ce660e2e0dc78b5c4", - "sha256:bfdf02789e3d197bd682a758cae0a4a18706566395fbe2803badcd1335e0173e" - ], - "index": "pypi", - "version": "==1.10.1" - }, "python-dateutil": { "hashes": [ "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", @@ -268,13 +158,6 @@ ], "version": "==2.8.0" }, - "python-json-logger": { - "hashes": [ - "sha256:30999d1d742ecf6645991a2ce9273188505e98b713ad63be06aabff47dd1b3c4", - "sha256:8205cfe7061715de5cd1b37e3565d5b97d0ac13b30ff3ee612554abb6093d640" - ], - "version": "==0.1.8" - }, "pytz": { "hashes": [ "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", @@ -297,71 +180,6 @@ ], "version": "==1.12.0" }, - "snowballstemmer": { - "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" - ], - "version": "==1.2.1" - }, - "sphinx": { - "hashes": [ - "sha256:230af939a2f678ab4f2a0a948c3b24a822a0d280821859caaefb750ef7413003", - "sha256:835c701420102a0a71ba2ed54a5bada2da6fd01263bf6dc8c5c80c798e27709c" - ], - "index": "pypi", - "version": "==2.0.0b1" - }, - "sphinx-rtd-theme": { - "hashes": [ - "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4", - "sha256:728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a" - ], - "index": "pypi", - "version": "==0.4.3" - }, - "sphinxcontrib-applehelp": { - "hashes": [ - "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897", - "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d" - ], - "version": "==1.0.1" - }, - "sphinxcontrib-devhelp": { - "hashes": [ - "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34", - "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981" - ], - "version": "==1.0.1" - }, - "sphinxcontrib-htmlhelp": { - "hashes": [ - "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422", - "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7" - ], - "version": "==1.0.2" - }, - "sphinxcontrib-jsmath": { - "hashes": [ - "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", - "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" - ], - "version": "==1.0.1" - }, - "sphinxcontrib-qthelp": { - "hashes": [ - "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20", - "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f" - ], - "version": "==1.0.2" - }, - "sphinxcontrib-serializinghtml": { - "hashes": [ - "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227", - "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768" - ], - "version": "==1.1.3" - }, "urllib3": { "hashes": [ "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", @@ -371,6 +189,13 @@ } }, "develop": { + "alabaster": { + "hashes": [ + "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", + "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" + ], + "version": "==0.7.12" + }, "appdirs": { "hashes": [ "sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92", @@ -399,13 +224,20 @@ ], "version": "==19.1.0" }, + "babel": { + "hashes": [ + "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", + "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + ], + "version": "==2.6.0" + }, "black": { "hashes": [ - "sha256:22158b89c1a6b4eb333a1e65e791a3f8b998cf3b11ae094adb2570f31f769a44", - "sha256:4b475bbd528acce094c503a3d2dbc2d05a4075f6d0ef7d9e7514518e14cc5191" + "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", + "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" ], "index": "pypi", - "version": "==18.6b4" + "version": "==19.3b0" }, "bleach": { "hashes": [ @@ -493,10 +325,10 @@ }, "identify": { "hashes": [ - "sha256:244e7864ef59f0c7c50c6db73f58564151d91345cd9b76ed793458953578cadd", - "sha256:8ff062f90ad4b09cfe79b5dfb7a12e40f19d2e68a5c9598a49be45f16aba7171" + "sha256:443f419ca6160773cbaf22dbb302b1e436a386f23129dbb5482b68a147c2eca9", + "sha256:bd7f15fe07112b713fb68fbdde3a34dd774d9062128f2c398104889f783f989d" ], - "version": "==1.4.1" + "version": "==1.4.2" }, "idna": { "hashes": [ @@ -505,6 +337,13 @@ ], "version": "==2.8" }, + "imagesize": { + "hashes": [ + "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", + "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" + ], + "version": "==1.1.0" + }, "importlib-metadata": { "hashes": [ "sha256:46fc60c34b6ed7547e2a723fc8de6dc2e3a1173f8423246b3ce497f064e9c3de", @@ -512,21 +351,53 @@ ], "version": "==0.9" }, - "importlib-resources": { + "isort": { "hashes": [ - "sha256:6e2783b2538bd5a14678284a3962b0660c715e5a0f10243fd5e00a4b5974f50b", - "sha256:d3279fd0f6f847cced9f7acc19bd3e5df54d34f93a2e7bb5f238f81545787078" + "sha256:01cb7e1ca5e6c5b3f235f0385057f70558b70d2f00320208825fa62887292f43", + "sha256:268067462aed7eb2a1e237fcb287852f22077de3fb07964e87e00f829eea2d1a" ], - "markers": "python_version < '3.7'", - "version": "==1.0.2" + "index": "pypi", + "version": "==4.3.17" }, - "isort": { + "jinja2": { + "hashes": [ + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" + ], + "version": "==2.10.1" + }, + "markupsafe": { "hashes": [ - "sha256:18c796c2cd35eb1a1d3f012a214a542790a1aed95e29768bdcb9f2197eccbd0b", - "sha256:96151fca2c6e736503981896495d344781b60d18bfda78dc11b290c6125ebdb6" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" ], - "index": "pypi", - "version": "==4.3.15" + "version": "==1.1.1" }, "more-itertools": { "hashes": [ @@ -542,6 +413,13 @@ ], "version": "==1.3.3" }, + "packaging": { + "hashes": [ + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + ], + "version": "==19.0" + }, "pkginfo": { "hashes": [ "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", @@ -558,11 +436,11 @@ }, "pre-commit": { "hashes": [ - "sha256:d3d69c63ae7b7584c4b51446b0b583d454548f9df92575b2fe93a68ec800c4d3", - "sha256:fc512f129b9526e35e80d656a16a31c198f584c4fce3a5c739045b5140584917" + "sha256:2576a2776098f3902ef9540a84696e8e06bf18a337ce43a6a889e7fa5d26c4c5", + "sha256:82f2f2d657d7f9280de9f927ae56886d60b9ef7f3714eae92d12713cd9cb9e11" ], "index": "pypi", - "version": "==1.14.4" + "version": "==1.15.2" }, "py": { "hashes": [ @@ -578,11 +456,19 @@ ], "version": "==2.3.1" }, + "pyparsing": { + "hashes": [ + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + ], + "version": "==2.4.0" + }, "pytest": { "hashes": [ "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], + "index": "pypi", "version": "==4.4.1" }, "pytest-cov": { @@ -593,6 +479,21 @@ "index": "pypi", "version": "==2.6.1" }, + "pytest-mock": { + "hashes": [ + "sha256:43ce4e9dd5074993e7c021bb1c22cbb5363e612a2b5a76bc6d956775b10758b7", + "sha256:5bf5771b1db93beac965a7347dc81c675ec4090cb841e49d9d34637a25c30568" + ], + "index": "pypi", + "version": "==1.10.4" + }, + "pytz": { + "hashes": [ + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" + ], + "version": "==2019.1" + }, "pyyaml": { "hashes": [ "sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c", @@ -631,6 +532,14 @@ ], "version": "==0.9.1" }, + "responses": { + "hashes": [ + "sha256:502d9c0c8008439cfcdef7e251f507fcfdd503b56e8c0c87c3c3e3393953f790", + "sha256:97193c0183d63fba8cd3a041c75464e4b09ea0aff6328800d1546598567dde0b" + ], + "index": "pypi", + "version": "==0.10.6" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -638,6 +547,71 @@ ], "version": "==1.12.0" }, + "snowballstemmer": { + "hashes": [ + "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", + "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + ], + "version": "==1.2.1" + }, + "sphinx": { + "hashes": [ + "sha256:423280646fb37944dd3c85c58fb92a20d745793a9f6c511f59da82fa97cd404b", + "sha256:de930f42600a4fef993587633984cc5027dedba2464bcf00ddace26b40f8d9ce" + ], + "index": "pypi", + "version": "==2.0.1" + }, + "sphinx-rtd-theme": { + "hashes": [ + "sha256:00cf895504a7895ee433807c62094cf1e95f065843bf3acd17037c3e9a2becd4", + "sha256:728607e34d60456d736cc7991fd236afb828b21b82f956c5ea75f94c8414040a" + ], + "index": "pypi", + "version": "==0.4.3" + }, + "sphinxcontrib-applehelp": { + "hashes": [ + "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897", + "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-devhelp": { + "hashes": [ + "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34", + "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-htmlhelp": { + "hashes": [ + "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422", + "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7" + ], + "version": "==1.0.2" + }, + "sphinxcontrib-jsmath": { + "hashes": [ + "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", + "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-qthelp": { + "hashes": [ + "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20", + "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f" + ], + "version": "==1.0.2" + }, + "sphinxcontrib-serializinghtml": { + "hashes": [ + "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227", + "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768" + ], + "version": "==1.1.3" + }, "strip-hints": { "hashes": [ "sha256:25a9cd0e88d0945e5b26f7ba808218d730b00392a89c6198acaedeae316db1cb" @@ -654,11 +628,11 @@ }, "tox": { "hashes": [ - "sha256:04f8f1aa05de8e76d7a266ccd14e0d665d429977cd42123bc38efa9b59964e9e", - "sha256:25ef928babe88c71e3ed3af0c464d1160b01fca2dd1870a5bb26c2dea61a17fc" + "sha256:1b166b93d2ce66bb7b253ba944d2be89e0c9d432d49eeb9da2988b4902a4684e", + "sha256:665cbdd99f5c196dd80d1d8db8c8cf5d48b1ae1f778bccd1bdf14d5aaf4ca0fc" ], "index": "pypi", - "version": "==3.7.0" + "version": "==3.9.0" }, "tox-pyenv": { "hashes": [ @@ -692,10 +666,10 @@ }, "virtualenv": { "hashes": [ - "sha256:6aebaf4dd2568a0094225ebbca987859e369e3e5c22dc7d52e5406d504890417", - "sha256:984d7e607b0a5d1329425dd8845bd971b957424b5ba664729fab51ab8c11bc39" + "sha256:15ee248d13e4001a691d9583948ad3947bcb8a289775102e4c4aa98a8b7a6d73", + "sha256:bfc98bb9b42a3029ee41b96dc00a34c2f254cbf7716bec824477b2c82741a5c4" ], - "version": "==16.4.3" + "version": "==16.5.0" }, "webencodings": { "hashes": [ @@ -711,19 +685,12 @@ ], "version": "==0.33.1" }, - "yolk3k": { - "hashes": [ - "sha256:cf8731dd0a9f7ef50b5dc253fe0174383e3fed295a653672aa918c059eef86ae" - ], - "index": "pypi", - "version": "==0.9" - }, "zipp": { "hashes": [ - "sha256:55ca87266c38af6658b84db8cfb7343cdb0bf275f93c7afaea0d8e7a209c7478", - "sha256:682b3e1c62b7026afe24eadf6be579fb45fec54c07ea218bded8092af07a68c4" + "sha256:139391b239594fd8b91d856bc530fbd2df0892b17dd8d98a91f018715954185f", + "sha256:8047e4575ce8d700370a3301bbfc972896a5845eb62dd535da395b86be95dfad" ], - "version": "==0.3.3" + "version": "==0.4.0" } } } diff --git a/README.md b/README.md index 22df9c8d57..3bf3b63822 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ Cognite Python SDK [![tox](https://img.shields.io/badge/tox-3.5%2B-blue.svg)](https://www.python.org/downloads/release/python-350/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) -Python SDK to ensure excellent user experience for developers and data scientists working with the Cognite Data Platform. +This is the Cognite Python SDK for developers and data scientists working with Cognite Data Fusion (CDF). +The package is tightly integrated with pandas, and helps you work easily and efficiently with data in Cognite Data +Fusion (CDF). ## Documentation * [SDK Documentation](https://cognite-docs.readthedocs-hosted.com/en/latest/) @@ -31,12 +33,19 @@ $ export COGNITE_API_KEY= On Windows, you can follows [these instructions](https://www.computerhope.com/issues/ch000549.htm) to set the API key as an environment variable. ## Installation +To install this package: ```bash $ pip install cognite-sdk ``` +To install this package without the pandas and NumPy support: +```bash +$ pip install cognite-sdk-core +``` + ## Examples -Examples on how to use the SDK can be found [here](https://github.com/cognitedata/cognite-python-docs) +For a collection of scripts and Jupyter Notebooks that explain how to perform various tasks in Cognite Data Fusion (CDF) +using Python, see the GitHub repository [here](https://github.com/cognitedata/cognite-python-docs) ## Changelog Wondering about upcoming or previous changes to the SDK? Take a look at the [CHANGELOG](https://github.com/cognitedata/cognite-sdk-python/blob/master/CHANGELOG.md). diff --git a/cognite/client/__init__.py b/cognite/client/__init__.py index 614507b341..ed2bcbc749 100644 --- a/cognite/client/__init__.py +++ b/cognite/client/__init__.py @@ -1,4 +1,3 @@ -from cognite.client.cognite_client import CogniteClient -from cognite.client.exceptions import APIError +from cognite.client._cognite_client import CogniteClient -__version__ = "0.13.3" +__version__ = "1.0.0a20" diff --git a/cognite/client/_api/__init__.py b/cognite/client/_api/__init__.py new file mode 100644 index 0000000000..a2497d80e8 --- /dev/null +++ b/cognite/client/_api/__init__.py @@ -0,0 +1 @@ +from typing import * diff --git a/cognite/client/_api/assets.py b/cognite/client/_api/assets.py new file mode 100644 index 0000000000..82b5327b4d --- /dev/null +++ b/cognite/client/_api/assets.py @@ -0,0 +1,536 @@ +import queue +import threading +from collections import OrderedDict +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import Asset, AssetFilter, AssetList, AssetUpdate +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils import _utils as utils + + +class AssetsAPI(APIClient): + _RESOURCE_PATH = "/assets" + _LIST_CLASS = AssetList + + def __call__( + self, + chunk_size: int = None, + name: str = None, + parent_ids: List[int] = None, + metadata: Dict[str, Any] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + root: bool = None, + external_id_prefix: str = None, + ) -> Generator[Union[Asset, AssetList], None, None]: + """Iterate over assets + + Fetches assets as they are iterated over, so you keep a limited number of assets in memory. + + Args: + chunk_size (int, optional): Number of assets to return in each chunk. Defaults to yielding one asset a time. + name (str): Name of asset. Often referred to as tag. + parent_ids (List[int]): No description. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + root (bool): filtered assets are root assets or not + external_id_prefix (str): External Id provided by client. Should be unique within the project + + Yields: + Union[Asset, AssetList]: yields Asset one by one if chunk is not specified, else AssetList objects. + """ + + filter = AssetFilter( + name, parent_ids, metadata, source, created_time, last_updated_time, root, external_id_prefix + ).dump(camel_case=True) + return self._list_generator(method="POST", chunk_size=chunk_size, filter=filter) + + def __iter__(self) -> Generator[Asset, None, None]: + """Iterate over assets + + Fetches assets as they are iterated over, so you keep a limited number of assets in memory. + + Yields: + Asset: yields Assets one by one. + """ + return self.__call__() + + def retrieve(self, id: Optional[int] = None, external_id: Optional[str] = None) -> Optional[Asset]: + """Retrieve a single asset by id. + + Args: + id (int, optional): ID + external_id (str, optional): External ID + + Returns: + Optional[Asset]: Requested asset or None if it does not exist. + + Examples: + + Get asset by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.retrieve(id=1) + + Get asset by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.retrieve(external_id="1") + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + return self._retrieve_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def retrieve_multiple(self, ids: Optional[List[int]] = None, external_ids: Optional[List[str]] = None) -> AssetList: + """Retrieve multiple assets by id. + + Args: + ids (List[int], optional): IDs + external_ids (List[str], optional): External IDs + + Returns: + AssetList: The requested assets. + + Examples: + + Get assets by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.retrieve_multiple(ids=[1, 2, 3]) + + Get assets by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.retrieve_multiple(external_ids=["abc", "def"]) + """ + utils.assert_type(ids, "id", [List], allow_none=True) + utils.assert_type(external_ids, "external_id", [List], allow_none=True) + return self._retrieve_multiple(ids=ids, external_ids=external_ids, wrap_ids=True) + + def list( + self, + name: str = None, + parent_ids: List[int] = None, + metadata: Dict[str, Any] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + root: bool = None, + external_id_prefix: str = None, + limit: int = 25, + ) -> AssetList: + """List assets + + Args: + name (str): Name of asset. Often referred to as tag. + parent_ids (List[int]): No description. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + root (bool): filtered assets are root assets or not + external_id_prefix (str): External Id provided by client. Should be unique within the project + limit (int, optional): Maximum number of assets to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + AssetList: List of requested assets + + Examples: + + List assets:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> asset_list = c.assets.list(limit=5) + + Iterate over assets:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for asset in c.assets: + ... asset # do something with the asset + + Iterate over chunks of assets to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for asset_list in c.assets(chunk_size=2500): + ... asset_list # do something with the assets + """ + filter = AssetFilter( + name, parent_ids, metadata, source, created_time, last_updated_time, root, external_id_prefix + ).dump(camel_case=True) + return self._list(method="POST", limit=limit, filter=filter) + + def create(self, asset: Union[Asset, List[Asset]]) -> Union[Asset, AssetList]: + """Create one or more assets. + + Args: + asset (Union[Asset, List[Asset]]): Asset or list of assets to create. + + Returns: + Union[Asset, AssetList]: Created asset(s) + + Examples: + + Create new assets:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import Asset + >>> c = CogniteClient() + >>> assets = [Asset(name="asset1"), Asset(name="asset2")] + >>> res = c.assets.create(assets) + """ + utils.assert_type(asset, "asset", [Asset, list]) + if isinstance(asset, Asset) or len(asset) <= self._CREATE_LIMIT: + return self._create_multiple(asset) + return _AssetPoster(asset, client=self).post() + + def delete(self, id: Union[int, List[int]] = None, external_id: Union[str, List[str]] = None) -> None: + """Delete one or more assets + + Args: + id (Union[int, List[int]): Id or list of ids + external_id (Union[str, List[str]]): External ID or list of exgernal ids + + Returns: + None + + Examples: + + Delete assets by id or external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.delete(id=[1,2,3], external_id="3") + """ + self._delete_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def update(self, item: Union[Asset, AssetUpdate, List[Union[Asset, AssetUpdate]]]) -> Union[Asset, AssetList]: + """Update one or more assets + + Args: + item (Union[Asset, AssetUpdate, List[Union[Asset, AssetUpdate]]]): Asset(s) to update + + Returns: + Union[Asset, AssetList]: Updated asset(s) + + Examples: + + Update an asset that you have fetched. This will perform a full update of the asset:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> asset = c.assets.retrieve(id=1) + >>> asset.description = "New description" + >>> res = c.assets.update(asset) + + Perform a partial update on a asset, updating the description and adding a new field to metadata:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import AssetUpdate + >>> c = CogniteClient() + >>> my_update = AssetUpdate(id=1).description.set("New description").metadata.add({"key": "value"}) + >>> res = c.assets.update(my_update) + """ + return self._update_multiple(items=item) + + def search( + self, name: str = None, description: str = None, filter: Union[AssetFilter, Dict] = None, limit: int = None + ) -> AssetList: + """Search for assets + + Args: + name (str): Fuzzy match on name. + description (str): Fuzzy match on description. + filter (Union[AssetFilter, Dict]): Filter to apply. Performs exact match on these fields. + limit (int): Maximum number of results to return. + + Returns: + AssetList: List of requested assets + + Examples: + + Search for assets:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.assets.search(name="some name") + """ + return self._search(search={"name": name, "description": description}, filter=filter, limit=limit) + + def retrieve_subtree(self, id: int = None, external_id: str = None, depth: int = None) -> AssetList: + """Retrieve the subtree for this asset up to a specified depth. + + Args: + id (int): Id of the root asset in the subtree. + external_id (str): External id of the root asset in the subtree. + depth (int): Retrieve assets up to this depth below the root asset in the subtree. Omit to get the entire + subtree. + + Returns: + AssetList: The requested assets sorted topologically. + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + asset = self.retrieve(id=id, external_id=external_id) + subtree = self._get_asset_subtree(AssetList([asset]), current_depth=0, depth=depth) + return AssetList(sorted(subtree, key=lambda a: a.depth), cognite_client=self._cognite_client) + + def _get_asset_subtree(self, assets: AssetList, current_depth: int, depth: int) -> AssetList: + subtree = assets + if depth is None or current_depth < depth: + children = self._get_children(assets) + if children: + subtree.extend(self._get_asset_subtree(children, current_depth + 1, depth)) + return subtree + + def _get_children(self, assets: AssetList) -> AssetList: + ids = [a.id for a in assets] + tasks = [] + chunk_size = 100 + for i in range(0, len(ids), chunk_size): + tasks.append({"parent_ids": ids[i : i + chunk_size], "limit": -1}) + res_list = utils.execute_tasks_concurrently(self.list, tasks=tasks, max_workers=self._max_workers).results + children = AssetList([]) + for res in res_list: + children.extend(res) + return children + + +class _AssetsFailedToPost: + def __init__(self, exc: Exception, assets: List[Asset]): + self.exc = exc + self.assets = assets + + +class _AssetPosterWorker(threading.Thread): + def __init__(self, client: AssetsAPI, request_queue: queue.Queue, response_queue: queue.Queue): + self.client = client + self.request_queue = request_queue + self.response_queue = response_queue + self.stop = False + super().__init__(daemon=True) + + def run(self): + request = None + try: + while not self.stop: + try: + request = self.request_queue.get(timeout=0.1) + except queue.Empty: + continue + assets = self.client.create(request) + self.response_queue.put(assets) + except Exception as e: + self.response_queue.put(_AssetsFailedToPost(e, request)) + + +class _AssetPoster: + def __init__(self, assets: List[Asset], client: AssetsAPI): + self._validate_asset_hierarchy(assets) + self.remaining_external_ids = OrderedDict() + self.remaining_external_ids_set = set() + self.external_id_to_asset = {} + + for asset in assets: + asset_copy = Asset(**asset.dump()) + external_id = asset.external_id + if external_id is None: + external_id = utils.random_string() + asset_copy.external_id = external_id + self.remaining_external_ids[external_id] = None + self.remaining_external_ids_set.add(external_id) + self.external_id_to_asset[external_id] = asset_copy + + self.client = client + + self.num_of_assets = len(self.remaining_external_ids) + self.external_ids_without_circular_deps = set() + self.external_id_to_children = {external_id: set() for external_id in self.remaining_external_ids} + self.external_id_to_descendent_count = {external_id: 0 for external_id in self.remaining_external_ids} + self.successfully_posted_external_ids = set() + self.posted_assets = set() + self.may_have_been_posted_assets = set() + self.not_posted_assets = set() + self.exception = None + + self.assets_remaining = ( + lambda: len(self.posted_assets) + len(self.may_have_been_posted_assets) + len(self.not_posted_assets) + < self.num_of_assets + ) + + self.request_queue = queue.Queue() + self.response_queue = queue.Queue() + + self._initialize() + + @staticmethod + def _validate_asset_hierarchy(assets) -> None: + external_ids = set([asset.external_id for asset in assets]) + external_ids_seen = set() + for asset in assets: + if asset.external_id: + if asset.external_id in external_ids_seen: + raise AssertionError("Duplicate external_id '{}' found".format(asset.external_id)) + external_ids_seen.add(asset.external_id) + + parent_ref = asset.parent_external_id + if parent_ref: + if parent_ref not in external_ids: + raise AssertionError("parent_external_id '{}' does not point to any asset".format(parent_ref)) + if asset.parent_id is not None: + raise AssertionError( + "An asset has both parent_id '{}' and parent_external_id '{}' set.".format( + asset.parent_id, asset.parent_external_id + ) + ) + + def _initialize(self): + root_assets = set() + for external_id in self.remaining_external_ids: + asset = self.external_id_to_asset[external_id] + if asset.parent_external_id is None or asset.parent_id is not None: + root_assets.add(asset) + elif asset.parent_external_id in self.external_id_to_children: + self.external_id_to_children[asset.parent_external_id].add(asset) + self._verify_asset_is_not_part_of_tree_with_circular_deps(asset) + + for root_asset in root_assets: + self._initialize_asset_to_descendant_count(root_asset) + + self.remaining_external_ids = self._sort_external_ids_by_descendant_count(self.remaining_external_ids) + + def _initialize_asset_to_descendant_count(self, asset): + for child in self.external_id_to_children[asset.external_id]: + self.external_id_to_descendent_count[asset.external_id] += 1 + self._initialize_asset_to_descendant_count( + child + ) + return self.external_id_to_descendent_count[asset.external_id] + + def _get_descendants(self, asset): + descendants = [] + for child in self.external_id_to_children[asset.external_id]: + descendants.append(child) + descendants.extend(self._get_descendants(child)) + return descendants + + def _verify_asset_is_not_part_of_tree_with_circular_deps(self, asset: Asset): + next_asset = asset + seen = {asset.external_id} + while next_asset.parent_external_id is not None: + next_asset = self.external_id_to_asset[next_asset.parent_external_id] + if next_asset.external_id in self.external_ids_without_circular_deps: + break + if next_asset.external_id not in seen: + seen.add(next_asset.external_id) + else: + raise AssertionError("The asset hierarchy has circular dependencies") + self.external_ids_without_circular_deps.update(seen) + + def _sort_external_ids_by_descendant_count(self, external_ids: OrderedDict) -> OrderedDict: + sorted_external_ids = sorted(external_ids, key=lambda x: self.external_id_to_descendent_count[x], reverse=True) + return OrderedDict({external_id: None for external_id in sorted_external_ids}) + + def _get_assets_unblocked_locally(self, asset: Asset, limit): + pq = utils.PriorityQueue() + pq.add(asset, self.external_id_to_descendent_count[asset.external_id]) + unblocked_descendents = set() + while pq: + if len(unblocked_descendents) == limit: + break + asset = pq.get() + unblocked_descendents.add(asset) + self.remaining_external_ids_set.remove(asset.external_id) + for child in self.external_id_to_children[asset.external_id]: + pq.add(child, self.external_id_to_descendent_count[child.external_id]) + return unblocked_descendents + + def _get_unblocked_assets(self) -> List[Set[Asset]]: + limit = self.client._CREATE_LIMIT + unblocked_assets_lists = [] + unblocked_assets_chunk = set() + for external_id in self.remaining_external_ids: + asset = self.external_id_to_asset[external_id] + parent_external_id = asset.parent_external_id + + if external_id in self.remaining_external_ids_set: + has_parent_id = asset.parent_id is not None + is_root = (not has_parent_id) and parent_external_id is None + is_unblocked = parent_external_id in self.successfully_posted_external_ids + if is_root or has_parent_id or is_unblocked: + unblocked_assets_chunk.update( + self._get_assets_unblocked_locally(asset, limit - len(unblocked_assets_chunk)) + ) + if len(unblocked_assets_chunk) == limit: + unblocked_assets_lists.append(unblocked_assets_chunk) + unblocked_assets_chunk = set() + if len(unblocked_assets_chunk) > 0: + unblocked_assets_lists.append(unblocked_assets_chunk) + + for unblocked_assets_chunk in unblocked_assets_lists: + for unblocked_asset in unblocked_assets_chunk: + del self.remaining_external_ids[unblocked_asset.external_id] + + return unblocked_assets_lists + + def run(self): + unblocked_assets_lists = self._get_unblocked_assets() + for unblocked_assets in unblocked_assets_lists: + self.request_queue.put(list(unblocked_assets)) + while self.assets_remaining(): + res = self.response_queue.get() + if isinstance(res, _AssetsFailedToPost): + if isinstance(res.exc, CogniteAPIError): + self.exception = res.exc + for asset in res.assets: + if res.exc.code >= 500: + self.may_have_been_posted_assets.add(asset) + elif res.exc.code >= 400: + self.not_posted_assets.add(asset) + for descendant in self._get_descendants(asset): + self.not_posted_assets.add(descendant) + else: + raise res.exc + else: + for asset in res: + self.posted_assets.add(asset) + self.successfully_posted_external_ids.add(asset.external_id) + unblocked_assets_lists = self._get_unblocked_assets() + for unblocked_assets in unblocked_assets_lists: + self.request_queue.put(list(unblocked_assets)) + if len(self.may_have_been_posted_assets) > 0 or len(self.not_posted_assets) > 0: + if isinstance(self.exception, CogniteAPIError): + raise CogniteAPIError( + message=self.exception.message, + code=self.exception.code, + x_request_id=self.exception.x_request_id, + missing=self.exception.missing, + duplicated=self.exception.duplicated, + successful=AssetList(list(self.posted_assets)), + unknown=AssetList(list(self.may_have_been_posted_assets)), + failed=AssetList(list(self.not_posted_assets)), + unwrap_fn=lambda a: a.external_id, + ) + raise self.exception + + def post(self): + workers = [] + for _ in range(self.client._max_workers): + worker = _AssetPosterWorker(self.client, self.request_queue, self.response_queue) + workers.append(worker) + worker.start() + + self.run() + + for worker in workers: + worker.stop = True + + return AssetList(sorted(self.posted_assets, key=lambda x: x.path)) diff --git a/cognite/client/_api/datapoints.py b/cognite/client/_api/datapoints.py new file mode 100644 index 0000000000..8d572f7170 --- /dev/null +++ b/cognite/client/_api/datapoints.py @@ -0,0 +1,821 @@ +import math +import threading +from collections import defaultdict, namedtuple +from datetime import datetime +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import Datapoints, DatapointsList, DatapointsQuery +from cognite.client.utils import _utils as utils + + +class DatapointsAPI(APIClient): + _RESOURCE_PATH = "/timeseries/data" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._DPS_LIMIT_AGG = 10000 + self._DPS_LIMIT = 100000 + + def retrieve( + self, + start: Union[int, str, datetime], + end: Union[int, str, datetime], + id: Union[int, List[int], Dict[str, Union[int, List[str]]], List[Dict[str, Union[int, List[str]]]]] = None, + external_id: Union[ + str, List[str], Dict[str, Union[int, List[str]]], List[Dict[str, Union[int, List[str]]]] + ] = None, + aggregates: List[str] = None, + granularity: str = None, + include_outside_points: bool = None, + limit: int = None, + ) -> Union[Datapoints, DatapointsList]: + """Get datapoints for one or more time series + + Args: + start (Union[int, str, datetime]): Inclusive start. + end (Union[int, str, datetime]): Exclusive end. + id (Union[int, List[int], Dict[str, Any], List[Dict[str, Any]]]: Id or list of ids. Can also be object + specifying aggregates. See example below. + external_id (Union[str, List[str], Dict[str, Any], List[Dict[str, Any]]]): External id or list of external + ids. Can also be object specifying aggregates. See example below. + aggregates (List[str]): List of aggregate functions to apply. + granularity (str): The granularity to fetch aggregates at. e.g. '1s', '2h', '10d'. + include_outside_points (bool): Whether or not to include outside points. + limit (int): Maximum number of datapoints to return for each time series. + + Returns: + Union[Datapoints, DatapointsList]: A Datapoints object containing the requested data, or a list of such objects. + + Examples: + + You can get specify the ids of the datapoints you wish to retrieve in a number of ways. In this example + we are using the time-ago format to get raw data for the time series with id 1:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> dps = c.datapoints.retrieve(id=1, start="2w-ago", end="now") + + We can also get aggregated values, such as average. Here we are getting daily averages for all of 2018 for + two different time series. Note that we arefetching them using their external ids:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> dps = c.datapoints.retrieve(external_id=["abc", "def"], + ... start=datetime(2018,1,1), + ... end=datetime(2019,1,1), + ... aggregates=["avg"], + ... granularity="1d") + + If you want different aggregates for different time series specify your ids like this:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> dps = c.datapoints.retrieve(id=[{"id": 1, "aggregates": ["avg"]}, + ... {"id": 1, "aggregates": ["min"]}], + ... external_id={"externalId": "1", "aggregates": ["max"]}, + ... start="1d-ago", end="now", granularity="1h") + """ + ts_items, is_single_id = self._process_ts_identifiers(id, external_id) + + dps_queries = [] + for ts_item in ts_items: + dps_queries.append(_DPQuery(start, end, ts_item, aggregates, granularity, include_outside_points, limit)) + + dps_list = _DatapointsFetcher(client=self).fetch(dps_queries) + + if is_single_id: + return dps_list[0] + return dps_list + + def retrieve_latest( + self, + id: Union[int, List[int]] = None, + external_id: Union[str, List[str]] = None, + before: Union[int, str, datetime] = None, + ) -> Union[Datapoints, DatapointsList]: + """Get the latest datapoint for one or more time series + + Args: + id (Union[int, List[int]]: Id or list of ids. + external_id (Union[str, List[str]): External id or list of external ids. + before: Union[int, str, datetime]: Get latest datapoint before this time. + + Returns: + Union[Datapoints, DatapointsList]: A Datapoints object containing the requested data, or a list of such objects. + + Examples: + + Getting the latest datapoint in a time series. This method returns a Datapoints object, so the datapoint will + be the first element:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.datapoints.retrieve_latest(id=1)[0] + + You can also get the first datapoint before a specific time:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.datapoints.retrieve_latest(id=1, before="2d-ago")[0] + + If you need the latest datapoint for multiple time series simply give a list of ids. Note that we are + using external ids here, but either will work:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.datapoints.retrieve_latest(external_id=["abc", "def"]) + >>> latest_abc = res[0][0] + >>> latest_def = res[1][0] + """ + before = utils.timestamp_to_ms(before) if before else None + all_ids = self._process_ids(id, external_id, wrap_ids=True) + is_single_id = self._is_single_identifier(id, external_id) + if before: + for id in all_ids: + id.update({"before": before}) + + res = self._post(url_path=self._RESOURCE_PATH + "/latest", json={"items": all_ids}).json()["items"] + if is_single_id: + return Datapoints._load(res[0], cognite_client=self._cognite_client) + return DatapointsList._load(res, cognite_client=self._cognite_client) + + def query(self, query: Union[DatapointsQuery, List[DatapointsQuery]]) -> Union[Datapoints, DatapointsList]: + """Get datapoints for one or more time series + + This method is different from get() in that you can specify different start times, end times, and granularities + for each requested time series. + + Args: + query (Union[DatapointsQuery, List[DatapointsQuery]): List of datapoint queries. + + Returns: + Union[Datapoints, DatapointsList]: A Datapoints object containing the requested data, or a list of such objects. + + Examples: + + This method is useful if you want to get multiple time series, but you want to specify different starts, + ends, or granularities for each. e.g.:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import DatapointsQuery + >>> c = CogniteClient() + >>> queries = [DatapointsQuery(id=1, start="2d-ago", end="now"), + ... DatapointsQuery(external_id="abc", + ... start="10d-ago", + ... end="now", + ... aggregates=["avg"], + ... granularity="1m")] + >>> res = c.datapoints.query(queries) + """ + is_single_query = False + if isinstance(query, DatapointsQuery): + is_single_query = True + query = [query] + + dps_queries = [] + for q in query: + utils.assert_exactly_one_of_id_or_external_id(q.id, q.external_id) + ts_items, _ = self._process_ts_identifiers(q.id, q.external_id) + dps_queries.append( + _DPQuery(q.start, q.end, ts_items[0], q.aggregates, q.granularity, q.include_outside_points, q.limit) + ) + results = _DatapointsFetcher(self).fetch(dps_queries) + if is_single_query: + return results[0] + return results + + def insert( + self, + datapoints: Union[ + List[Dict[Union[int, float, datetime], Union[int, float, str]]], + List[Tuple[Union[int, float, datetime], Union[int, float, str]]], + ], + id: int = None, + external_id: str = None, + ) -> None: + """Insert datapoints into a time series + + Timestamps can be represented as milliseconds since epoch or datetime objects. + + Args: + datapoints(Union[List[Dict, Tuple]]): The datapoints you wish to insert. Can either be a list of tuples or + a list of dictionaries. See examples below. + id (int): Id of time series to insert datapoints into. + external_id (str): External id of time series to insert datapoint into. + + Returns: + None + + Examples: + + Your datapoints can be a list of tuples where the first element is the timestamp and the second element is + the value:: + + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> # with datetime objects + >>> datapoints = [(datetime(2018,1,1), 1000), (datetime(2018,1,2), 2000)] + >>> res1 = c.datapoints.insert(datapoints, id=1) + >>> # with ms since epoch + >>> datapoints = [(150000000000, 1000), (160000000000, 2000)] + >>> res2 = c.datapoints.insert(datapoints, id=2) + + Or they can be a list of dictionaries:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> # with datetime objects + >>> datapoints = [{"timestamp": datetime(2018,1,1), "value": 1000}, + ... {"timestamp": datetime(2018,1,2), "value": 2000}] + >>> res1 = c.datapoints.insert(datapoints, external_id="abc") + >>> # with ms since epoch + >>> datapoints = [{"timestamp": 150000000000, "value": 1000}, + ... {"timestamp": 160000000000, "value": 2000}] + >>> res2 = c.datapoints.insert(datapoints, external_id="def") + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + datapoints = self._validate_and_format_datapoints(datapoints) + utils.assert_timestamp_not_in_1970(datapoints[0]["timestamp"]) + post_dps_object = self._process_ids(id, external_id, wrap_ids=True)[0] + post_dps_object.update({"datapoints": datapoints}) + self._insert_datapoints_concurrently([post_dps_object]) + + def insert_multiple(self, datapoints: List[Dict[str, Union[str, int, List]]]) -> None: + """Insert datapoints into multiple time series + + Args: + datapoints (List[Dict]): The datapoints you wish to insert along with the ids of the time series. + See examples below. + + Returns: + None + + Examples: + + Your datapoints can be a list of tuples where the first element is the timestamp and the second element is + the value:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + + >>> datapoints = [] + >>> # with datetime objects and id + >>> datapoints.append({"id": 1, "datapoints": [(datetime(2018,1,1), 1000), (datetime(2018,1,2), 2000)]}) + >>> # with ms since epoch and externalId + >>> datapoints.append({"externalId": 1, "datapoints": [(150000000000, 1000), (160000000000, 2000)]}) + + >>> res = c.datapoints.insert_multiple(datapoints) + + Or they can be a list of dictionaries:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + + >>> datapoints = [] + >>> # with datetime objects and external id + >>> datapoints.append({"externalId": "1", "datapoints": [{"timestamp": datetime(2018,1,1), "value": 1000}, + ... {"timestamp": datetime(2018,1,2), "value": 2000}]}) + >>> # with ms since epoch and id + >>> datapoints.append({"id": 1, "datapoints": [{"timestamp": 150000000000, "value": 1000}, + ... {"timestamp": 160000000000, "value": 2000}]}) + + >>> res = c.datapoints.insert_multiple(datapoints) + """ + valid_dps_objects = [] + for dps_object in datapoints: + for key in dps_object: + if key not in ("id", "externalId", "datapoints"): + raise AssertionError( + "Invalid key '{}' in datapoints. Must contain 'datapoints', and 'id' or 'externalId".format(key) + ) + valid_dps_object = dps_object.copy() + valid_dps_object["datapoints"] = self._validate_and_format_datapoints(dps_object["datapoints"]) + valid_dps_objects.append(valid_dps_object) + self._insert_datapoints_concurrently(valid_dps_objects) + + def delete_range( + self, start: Union[int, str, datetime], end: Union[int, str, datetime], id: int = None, external_id: str = None + ) -> None: + """Delete a range of datapoints from a time series. + + Args: + start (Union[int, str, datetime]): Inclusive start of delete range + end (Union[int, str, datetime]): Exclusvie end of delete range + id (int): Id of time series to delete data from + external_id (str): External id of time series to delete data from + + Returns: + None + + Examples: + + Deleting the last week of data from a time series:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.datapoints.delete_range(start="1w-ago", end="now", id=1) + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + start = utils.timestamp_to_ms(start) + end = utils.timestamp_to_ms(end) + assert end > start, "end must be larger than start" + + delete_dps_object = self._process_ids(id, external_id, wrap_ids=True)[0] + delete_dps_object.update({"inclusiveBegin": start, "exclusiveEnd": end}) + self._delete_datapoints_ranges([delete_dps_object]) + + def delete_ranges(self, ranges: List[Dict[str, Any]]) -> None: + """Delete a range of datapoints from multiple time series. + + Args: + ranges (List[Dict[str, Any]]): The ids an ranges to delete. See examples below. + + Returns: + None + + Examples: + + Each element in the list ranges must be specify either id or externalId, and a range:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> ranges = [{"id": 1, "start": "2d-ago", "end": "now"}, + ... {"externalId": "abc", "start": "2d-ago", "end": "now"}] + >>> res = c.datapoints.delete_ranges(ranges) + """ + valid_ranges = [] + for range in ranges: + for key in range: + if key not in ("id", "externalId", "start", "end"): + raise AssertionError( + "Invalid key '{}' in range. Must contain 'start', 'end', and 'id' or 'externalId".format(key) + ) + id = range.get("id") + external_id = range.get("externalId") + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + valid_range = self._process_ids(id, external_id, wrap_ids=True)[0] + valid_range.update({"inclusiveBegin": range["start"], "exclusiveEnd": range["end"]}) + valid_ranges.append(valid_range) + self._delete_datapoints_ranges(valid_ranges) + + def _delete_datapoints_ranges(self, delete_range_objects): + self._post(url_path=self._RESOURCE_PATH + "/delete", json={"items": delete_range_objects}) + + def retrieve_dataframe( + self, + start: Union[int, str, datetime], + end: Union[int, str, datetime], + aggregates: List[str], + granularity: str, + id: Union[int, List[int], Dict[str, Union[int, List[str]]], List[Dict[str, Union[int, List[str]]]]] = None, + external_id: Union[ + str, List[str], Dict[str, Union[int, List[str]]], List[Dict[str, Union[int, List[str]]]] + ] = None, + limit: int = None, + ): + """Get a pandas dataframe describing the requested data. + + Args: + start (Union[int, str, datetime]): Inclusive start. + end (Union[int, str, datetime]): Exclusive end. + aggregates (List[str]): List of aggregate functions to apply. + granularity (str): The granularity to fetch aggregates at. e.g. '1s', '2h', '10d'. + id (Union[int, List[int], Dict[str, Any], List[Dict[str, Any]]]: Id or list of ids. Can also be object + specifying aggregates. See example below. + external_id (Union[str, List[str], Dict[str, Any], List[Dict[str, Any]]]): External id or list of external + ids. Can also be object specifying aggregates. See example below. + limit (int): Maximum number of datapoints to return for each time series. + + Returns: + pandas.DataFrame: The requested dataframe + + Examples: + + Get a pandas dataframe:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> df = c.datapoints.retrieve_dataframe(id=[1,2,3], start="2w-ago", end="now", + ... aggregates=["average"], granularity="1h") + """ + pd = utils.local_import("pandas") + id_df = pd.DataFrame() + external_id_df = pd.DataFrame() + if id is not None: + id_df = self.retrieve( + id=id, start=start, end=end, aggregates=aggregates, granularity=granularity, limit=limit + ).to_pandas(column_names="id") + if external_id is not None: + external_id_df = self.retrieve( + external_id=external_id, + start=start, + end=end, + aggregates=aggregates, + granularity=granularity, + limit=limit, + ).to_pandas() + return pd.concat([id_df, external_id_df], axis="columns") + + def insert_dataframe(self, dataframe): + """Insert a dataframe. + + The index of the dataframe must contain the timestamps. The names of the remaining columns specify the ids of + the time series to which column contents will be written. + + Said time series must already exist. + + Args: + dataframe (pandas.DataFrame): Pandas DataFrame Object containing the time series. + + Returns: + None + + Examples: + Post a dataframe with white noise:: + + >>> import numpy as np + >>> import pandas as pd + >>> from cognite.client import CogniteClient + >>> from datetime import datetime, timedelta + >>> + >>> c = CogniteClient() + >>> ts_id = 123 + >>> start = datetime(2018, 1, 1) + >>> # The scaling by 1000 is important: timestamp() returns seconds + >>> x = pd.DatetimeIndex([start + timedelta(days=d) for d in range(100)]) + >>> y = np.random.normal(0, 1, 100) + >>> df = pd.DataFrame({ts_id: y}, index=x) + >>> res = c.datapoints.insert_dataframe(df) + """ + assert not dataframe.isnull().values.any(), "Dataframe contains NaNs. Remove them in order to insert the data." + dps = [] + for col in dataframe.columns: + dps.append( + { + "id": int(col), + "datapoints": list( + zip(dataframe.index.values.astype("datetime64[ms]").astype("int64").tolist(), dataframe[col]) + ), + } + ) + self.insert_multiple(dps) + + @staticmethod + def _process_ts_identifiers(ids, external_ids) -> Tuple[List[Dict], bool]: + is_list = False + items = [] + + if isinstance(ids, List): + is_list = True + for item in ids: + items.append(DatapointsAPI._process_single_ts_item(item, False)) + elif ids is None: + pass + else: + items.append(DatapointsAPI._process_single_ts_item(ids, False)) + + if isinstance(external_ids, List): + is_list = True + for item in external_ids: + items.append(DatapointsAPI._process_single_ts_item(item, True)) + elif external_ids is None: + pass + else: + items.append(DatapointsAPI._process_single_ts_item(external_ids, True)) + + return items, not is_list and len(items) == 1 + + @staticmethod + def _process_single_ts_item(item, external: bool): + item_type = "externalId" if external else "id" + id_type = str if external else int + if isinstance(item, id_type): + return {item_type: item} + elif isinstance(item, Dict): + for key in item: + if not key in [item_type, "aggregates"]: + raise ValueError("Unknown key '{}' in {} dict argument".format(key, item_type)) + if not item_type in item: + raise ValueError( + "When passing a dict to the {} argument, '{}' must be specified.".format(item_type, item_type) + ) + return item + raise TypeError("Invalid type '{}' for argument '{}'".format(type(item), item_type)) + + def _insert_datapoints_concurrently(self, post_dps_objects: List[Dict[str, Any]]): + tasks = [] + for dps_object in post_dps_objects: + for i in range(0, len(dps_object["datapoints"]), self._DPS_LIMIT): + dps_object_chunk = dps_object.copy() + dps_object_chunk["datapoints"] = dps_object["datapoints"][i : i + self._DPS_LIMIT] + tasks.append(([dps_object_chunk],)) + utils.execute_tasks_concurrently(self._insert_datapoints, tasks, max_workers=self._max_workers) + + def _insert_datapoints(self, post_dps_objects: List[Dict[str, Any]]): + self._post(url_path=self._RESOURCE_PATH, json={"items": post_dps_objects}) + + @staticmethod + def _validate_and_format_datapoints( + datapoints: Union[ + List[Dict[Union[int, float, datetime], Union[int, float, str]]], + List[Tuple[Union[int, float, datetime], Union[int, float, str]]], + ], + ) -> List[Dict[str, int]]: + utils.assert_type(datapoints, "datapoints", [list]) + assert len(datapoints) > 0, "No datapoints provided" + utils.assert_type(datapoints[0], "datapoints element", [tuple, dict]) + + valid_datapoints = [] + if isinstance(datapoints[0], tuple): + valid_datapoints = [{"timestamp": utils.timestamp_to_ms(t), "value": v} for t, v in datapoints] + elif isinstance(datapoints[0], dict): + for dp in datapoints: + assert "timestamp" in dp, "A datapoint is missing the 'timestamp' key" + assert "value" in dp, "A datapoint is missing the 'value' key" + valid_datapoints.append({"timestamp": utils.timestamp_to_ms(dp["timestamp"]), "value": dp["value"]}) + return valid_datapoints + + +_DPWindow = namedtuple("_DPWindow", ["start", "end"]) + + +class _DPQuery: + def __init__(self, start, end, ts_item, aggregates, granularity, include_outside_points, limit): + self.start = utils.timestamp_to_ms(start) + self.end = utils.timestamp_to_ms(end) + self.ts_item = ts_item + self.aggregates = aggregates + self.granularity = granularity + self.include_outside_points = include_outside_points + self.limit = limit + self.dps_result = None + + def as_tuple(self): + return ( + self.start, + self.end, + self.ts_item, + self.aggregates, + self.granularity, + self.include_outside_points, + self.limit, + ) + + +class _DatapointsFetcher: + def __init__(self, client: DatapointsAPI): + self.client = client + self.id_to_datapoints = defaultdict(lambda: Datapoints()) + self.id_to_finalized_query = {} + self.lock = threading.Lock() + + def fetch(self, dps_queries: List[_DPQuery]) -> DatapointsList: + self._preprocess_queries(dps_queries) + self._fetch_datapoints(dps_queries) + dps_list = self._get_dps_results() + dps_list = self._sort_dps_list_by_query_order(dps_list, dps_queries) + return dps_list + + def _preprocess_queries(self, queries: List[_DPQuery]): + for query in queries: + new_start = utils.timestamp_to_ms(query.start) + new_end = utils.timestamp_to_ms(query.end) + if query.aggregates: + new_start = self._align_with_granularity_unit(new_start, query.granularity) + new_end = self._align_with_granularity_unit(new_end, query.granularity) + query.start = new_start + query.end = new_end + + def _get_dps_results(self): + dps_list = DatapointsList([], cognite_client=self.client._cognite_client) + for id, dps in self.id_to_datapoints.items(): + finalized_query = self.id_to_finalized_query[id] + if finalized_query.include_outside_points: + dps = self._remove_duplicates(dps) + if finalized_query.limit and len(dps) > finalized_query.limit: + dps = dps[: finalized_query.limit] + dps_list.append(dps) + return dps_list + + def _sort_dps_list_by_query_order(self, dps_list: DatapointsList, queries: List[_DPQuery]): + order = {} + for i, q in enumerate(queries): + identifier = utils.unwrap_identifer(q.ts_item) + order[identifier] = i + + def custom_sort_order(item): + if item.id in order: + return order[item.id] + return order[item.external_id] + + return DatapointsList(sorted(dps_list, key=custom_sort_order)) + + def _fetch_datapoints(self, dps_queries: List[_DPQuery]): + tasks_summary = utils.execute_tasks_concurrently( + self._fetch_dps_initial_and_return_remaining_queries, + [(q,) for q in dps_queries], + max_workers=self.client._max_workers, + ) + if tasks_summary.exceptions: + raise tasks_summary.exceptions[0] + + remaining_queries = tasks_summary.joined_results() + if len(remaining_queries) > 0: + self._fetch_datapoints_for_remaining_queries(remaining_queries) + + def _fetch_dps_initial_and_return_remaining_queries(self, query: _DPQuery) -> List[_DPQuery]: + request_limit = self.client._DPS_LIMIT if query.aggregates is None else self.client._DPS_LIMIT_AGG + if query.limit is not None and query.limit <= request_limit: + query.dps_result = self._get_datapoints_with_paging(*query.as_tuple()) + self._store_finalized_query(query) + return [] + + user_limit = query.limit + query.limit = None + query.dps_result = self._get_datapoints(*query.as_tuple()) + query.limit = user_limit + + self._store_finalized_query(query) + + dps_in_first_query = len(query.dps_result) + if dps_in_first_query < request_limit: + return [] + + if user_limit: + user_limit -= dps_in_first_query + next_start_offset = utils.granularity_to_ms(query.granularity) if query.granularity else 1 + query.start = query.dps_result[-1].timestamp + next_start_offset + queries = self._split_query_into_windows(query.dps_result.id, query, request_limit, user_limit) + + return queries + + def _fetch_datapoints_for_remaining_queries(self, queries: List[_DPQuery]): + tasks_summary = utils.execute_tasks_concurrently( + self._get_datapoints_with_paging, [q.as_tuple() for q in queries], max_workers=self.client._max_workers + ) + if tasks_summary.exceptions: + raise tasks_summary.exceptions[0] + res_list = tasks_summary.results + for i, res in enumerate(res_list): + queries[i].dps_result = res + self._store_finalized_query(queries[i]) + + def _store_finalized_query(self, query: _DPQuery): + with self.lock: + self.id_to_datapoints[query.dps_result.id]._insert(query.dps_result) + self.id_to_finalized_query[query.dps_result.id] = query + + @staticmethod + def _align_with_granularity_unit(ts: int, granularity: str): + gms = utils.granularity_unit_to_ms(granularity) + if ts % gms == 0: + return ts + return ts - (ts % gms) + gms + + def _split_query_into_windows(self, id: int, query: _DPQuery, request_limit, user_limit): + windows = self._get_windows(id, query.start, query.end, query.granularity, request_limit, user_limit) + + return [ + _DPQuery( + w.start, + w.end, + query.ts_item, + query.aggregates, + query.granularity, + query.include_outside_points, + query.limit, + ) + for w in windows + ] + + def _get_windows(self, id, start, end, granularity, request_limit, user_limit): + count_granularity = "1d" + if granularity and utils.granularity_to_ms("1d") < utils.granularity_to_ms(granularity): + count_granularity = granularity + res = self._get_datapoints_with_paging( + start=start, + end=end, + ts_item={"id": id}, + aggregates=["count"], + granularity=count_granularity, + include_outside_points=False, + limit=None, + ) + counts = list(zip(res.timestamp, res.count)) + windows = [] + total_count = 0 + current_window_count = 0 + window_start = start + granularity_ms = utils.granularity_to_ms(granularity) if granularity else None + agg_count = lambda count: int( + min(math.ceil(utils.granularity_to_ms(count_granularity) / granularity_ms), count) + ) + for i, (ts, count) in enumerate(counts): + if i < len(counts) - 1: + next_timestamp = counts[i + 1][0] + next_raw_count = counts[i + 1][1] + next_count = next_raw_count if granularity is None else agg_count(next_raw_count) + else: + next_timestamp = end + next_count = 0 + current_count = count if granularity is None else agg_count(count) + total_count += current_count + current_window_count += current_count + if current_window_count + next_count > request_limit or i == len(counts) - 1: + window_end = next_timestamp + if granularity: + window_end = self._align_window_end(start, next_timestamp, granularity) + windows.append(_DPWindow(window_start, window_end)) + window_start = window_end + current_window_count = 0 + if user_limit and total_count >= user_limit: + break + return windows + + @staticmethod + def _align_window_end(start: int, end: int, granularity: str): + gms = utils.granularity_to_ms(granularity) + diff = end - start + end -= diff % gms + return end + + @staticmethod + def _remove_duplicates(dps_object: Datapoints) -> Datapoints: + frequencies = defaultdict(lambda: [0, []]) + for i, timestamp in enumerate(dps_object.timestamp): + frequencies[timestamp][0] += 1 + frequencies[timestamp][1].append(i) + + indices_to_remove = [] + for timestamp, freq in frequencies.items(): + if freq[0] > 1: + indices_to_remove += freq[1][1:] + + dps_object_without_duplicates = Datapoints(id=dps_object.id, external_id=dps_object.external_id) + for attr, values in dps_object._get_non_empty_data_fields(): + filtered_values = [elem for i, elem in enumerate(values) if i not in indices_to_remove] + setattr(dps_object_without_duplicates, attr, filtered_values) + + return dps_object_without_duplicates + + def _get_datapoints_with_paging( + self, + start: int, + end: int, + ts_item: Dict[str, Any], + aggregates: List[str], + granularity: str, + include_outside_points: bool, + limit: int, + ) -> Datapoints: + per_request_limit = self.client._DPS_LIMIT_AGG if aggregates else self.client._DPS_LIMIT + limit_next_request = per_request_limit + next_start = start + datapoints = Datapoints() + all_datapoints = Datapoints() + while ( + (len(all_datapoints) == 0 or len(datapoints) == per_request_limit) + and end > next_start + and len(all_datapoints) < (limit or float("inf")) + ): + datapoints = self._get_datapoints( + next_start, end, ts_item, aggregates, granularity, include_outside_points, limit_next_request + ) + if len(datapoints) == 0: + break + + if limit: + remaining_datapoints = limit - len(datapoints) + if remaining_datapoints < per_request_limit: + limit_next_request = remaining_datapoints + latest_timestamp = int(datapoints.timestamp[-1]) + next_start = latest_timestamp + (utils.granularity_to_ms(granularity) if granularity else 1) + all_datapoints._insert(datapoints) + return all_datapoints + + def _get_datapoints( + self, + start: int, + end: int, + ts_item: Dict[str, Any], + aggregates: List[str], + granularity: str, + include_outside_points: bool, + limit: int, + ) -> Datapoints: + payload = { + "items": [ts_item], + "start": start, + "end": end, + "aggregates": aggregates, + "granularity": granularity, + "includeOutsidePoints": include_outside_points, + "limit": limit or (self.client._DPS_LIMIT_AGG if aggregates else self.client._DPS_LIMIT), + } + res = self.client._post(self.client._RESOURCE_PATH + "/list", json=payload).json()["items"][0] + aggs = ts_item.get("aggregates", aggregates) + expected_fields = [a for a in aggs] if aggs is not None else ["value"] + dps = Datapoints._load(res, expected_fields, cognite_client=self.client._cognite_client) + return dps diff --git a/cognite/client/_api/events.py b/cognite/client/_api/events.py new file mode 100644 index 0000000000..c6ef16f3fa --- /dev/null +++ b/cognite/client/_api/events.py @@ -0,0 +1,282 @@ +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import Event, EventFilter, EventList, EventUpdate +from cognite.client.utils import _utils as utils + + +class EventsAPI(APIClient): + _RESOURCE_PATH = "/events" + _LIST_CLASS = EventList + + def __call__( + self, + chunk_size: int = None, + start_time: Dict[str, Any] = None, + end_time: Dict[str, Any] = None, + type: str = None, + subtype: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + external_id_prefix: str = None, + ) -> Generator[Union[Event, EventList], None, None]: + """Iterate over events + + Fetches events as they are iterated over, so you keep a limited number of events in memory. + + Args: + chunk_size (int, optional): Number of events to return in each chunk. Defaults to yielding one event a time. + start_time (Dict[str, Any]): Range between two timestamps + end_time (Dict[str, Any]): Range between two timestamps + type (str): Type of the event, e.g 'failure'. + subtype (str): Subtype of the event, e.g 'electrical'. + metadata (Dict[str, Any]): Customizable extra data about the event. String key -> String value. + asset_ids (List[int]): Asset IDs of related equipments that this event relates to. + source (str): The source of this event. + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project + + Yields: + Union[Event, EventList]: yields Event one by one if chunk is not specified, else EventList objects. + """ + filter = EventFilter( + start_time=start_time, + end_time=end_time, + metadata=metadata, + asset_ids=asset_ids, + source=source, + created_time=created_time, + last_updated_time=last_updated_time, + external_id_prefix=external_id_prefix, + type=type, + subtype=subtype, + ).dump(camel_case=True) + return self._list_generator(method="POST", chunk_size=chunk_size, filter=filter) + + def __iter__(self) -> Generator[Event, None, None]: + """Iterate over events + + Fetches events as they are iterated over, so you keep a limited number of events in memory. + + Yields: + Event: yields Events one by one. + """ + return self.__call__() + + def retrieve(self, id: Optional[int] = None, external_id: Optional[str] = None) -> Optional[Event]: + """Retrieve a single event by id. + + Args: + id (int, optional): ID + external_id (str, optional): External ID + + Returns: + Optional[Event]: Requested event or None if it does not exist. + + Examples: + + Get event by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.retrieve(id=1) + + Get event by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.retrieve(external_id="1") + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + return self._retrieve_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def retrieve_multiple(self, ids: Optional[List[int]] = None, external_ids: Optional[List[str]] = None) -> EventList: + """Retrieve multiple events by id. + + Args: + ids (List[int], optional): IDs + external_ids (List[str], optional): External IDs + + Returns: + EventList: The requested events. + + Examples: + + Get events by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.retrieve_multiple(ids=[1, 2, 3]) + + Get events by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.retrieve_multiple(external_ids=["abc", "def"]) + """ + utils.assert_type(ids, "id", [List], allow_none=True) + utils.assert_type(external_ids, "external_id", [List], allow_none=True) + return self._retrieve_multiple(ids=ids, external_ids=external_ids, wrap_ids=True) + + def list( + self, + start_time: Dict[str, Any] = None, + end_time: Dict[str, Any] = None, + type: str = None, + subtype: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + external_id_prefix: str = None, + limit: int = 25, + ) -> EventList: + """List events + + Args: + start_time (Dict[str, Any]): Range between two timestamps + end_time (Dict[str, Any]): Range between two timestamps + type (str): Type of the event, e.g 'failure'. + subtype (str): Subtype of the event, e.g 'electrical'. + metadata (Dict[str, Any]): Customizable extra data about the event. String key -> String value. + asset_ids (List[int]): Asset IDs of related equipments that this event relates to. + source (str): The source of this event. + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project + limit (int, optional): Maximum number of events to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + EventList: List of requested events + + Examples: + + List events and filter on max start time:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> file_list = c.events.list(limit=5, start_time={"max": 1500000000}) + + Iterate over events:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for event in c.events: + ... event # do something with the event + + Iterate over chunks of events to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for event_list in c.events(chunk_size=2500): + ... event_list # do something with the files + """ + filter = EventFilter( + start_time=start_time, + end_time=end_time, + metadata=metadata, + asset_ids=asset_ids, + source=source, + created_time=created_time, + last_updated_time=last_updated_time, + external_id_prefix=external_id_prefix, + type=type, + subtype=subtype, + ).dump(camel_case=True) + return self._list(method="POST", limit=limit, filter=filter) + + def create(self, event: Union[Event, List[Event]]) -> Union[Event, EventList]: + """Create one or more events. + + Args: + event (Union[Event, List[Event]]): Event or list of events to create. + + Returns: + Union[Event, EventList]: Created event(s) + + Examples: + + Create new events:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import Event + >>> c = CogniteClient() + >>> events = [Event(start_time=0, end_time=1), Event(start_time=2, end_time=3)] + >>> res = c.events.create(events) + """ + return self._create_multiple(items=event) + + def delete(self, id: Union[int, List[int]] = None, external_id: Union[str, List[str]] = None) -> None: + """Delete one or more events + + Args: + id (Union[int, List[int]): Id or list of ids + external_id (Union[str, List[str]]): External ID or list of external ids + + Returns: + None + Examples: + + Delete events by id or external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.delete(id=[1,2,3], external_id="3") + """ + self._delete_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def update(self, item: Union[Event, EventUpdate, List[Union[Event, EventUpdate]]]) -> Union[Event, EventList]: + """Update one or more events + + Args: + item (Union[Event, EventUpdate, List[Union[Event, EventUpdate]]]): Event(s) to update + + Returns: + Union[Event, EventList]: Updated event(s) + + Examples: + + Update an event that you have fetched. This will perform a full update of the event:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> event = c.events.retrieve(id=1) + >>> event.description = "New description" + >>> res = c.events.update(event) + + Perform a partial update on a event, updating the description and adding a new field to metadata:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import EventUpdate + >>> c = CogniteClient() + >>> my_update = EventUpdate(id=1).description.set("New description").metadata.add({"key": "value"}) + >>> res = c.events.update(my_update) + """ + return self._update_multiple(items=item) + + def search(self, description: str = None, filter: Union[EventFilter, Dict] = None, limit: int = None) -> EventList: + """Search for events + + Args: + description (str): Fuzzy match on description. + filter (Union[EventFilter, Dict]): Filter to apply. Performs exact match on these fields. + limit (int): Maximum number of results to return. + + Returns: + EventList: List of requested events + + Examples: + + Search for events:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.events.search(description="some description") + """ + return self._search(search={"description": description}, filter=filter, limit=limit) diff --git a/cognite/client/_api/files.py b/cognite/client/_api/files.py new file mode 100644 index 0000000000..43dfdffd52 --- /dev/null +++ b/cognite/client/_api/files.py @@ -0,0 +1,509 @@ +import os +import queue +import threading +from typing import * +from typing import Dict, List + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import FileMetadata, FileMetadataFilter, FileMetadataList, FileMetadataUpdate +from cognite.client.utils import _utils as utils + + +class FilesAPI(APIClient): + _RESOURCE_PATH = "/files" + _LIST_CLASS = FileMetadataList + + def __call__( + self, + chunk_size: int = None, + name: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + uploaded_time: Dict[str, Any] = None, + external_id_prefix: str = None, + uploaded: bool = None, + ) -> Generator[Union[FileMetadata, FileMetadataList], None, None]: + """Iterate over files + + Fetches file metadata objects as they are iterated over, so you keep a limited number of metadata objects in memory. + + Args: + chunk_size (int, optional): Number of files to return in each chunk. Defaults to yielding one event a time. + name (str): Name of the file. + mime_type (str): File type. E.g. text/plain, application/pdf, .. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): Only include files that reference these specific asset IDs. + source (str): The source of this event. + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + uploaded_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project. + uploaded (bool): Whether or not the actual file is uploaded. This field is returned only by the API, it has no effect in a post body. + + Yields: + Union[FileMetadata, FileMetadataList]: yields FileMetadata one by one if chunk is not specified, else FileMetadataList objects. + """ + filter = FileMetadataFilter( + name, + mime_type, + metadata, + asset_ids, + source, + created_time, + last_updated_time, + uploaded_time, + external_id_prefix, + uploaded, + ).dump(camel_case=True) + return self._list_generator(method="POST", chunk_size=chunk_size, filter=filter) + + def __iter__(self) -> Generator[FileMetadata, None, None]: + """Iterate over files + + Fetches file metadata objects as they are iterated over, so you keep a limited number of metadata objects in memory. + + Yields: + FileMetadata: yields Files one by one. + """ + return self.__call__() + + def retrieve(self, id: Optional[int] = None, external_id: Optional[str] = None) -> Optional[FileMetadata]: + """Retrieve a single file metadata by id. + + Args: + id (int, optional): ID + external_id (str, optional): External ID + + Returns: + Optional[FileMetadata]: Requested file metadata or None if it does not exist. + + Examples: + + Get file metadata by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.retrieve(id=1) + + Get file metadata by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.retrieve(external_id="1") + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + return self._retrieve_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def retrieve_multiple( + self, ids: Optional[List[int]] = None, external_ids: Optional[List[str]] = None + ) -> FileMetadataList: + """Retrieve multiple file metadatas by id. + + Args: + ids (List[int], optional): IDs + external_ids (List[str], optional): External IDs + + Returns: + FileMetadataList: The requested file metadatas. + + Examples: + + Get file metadatas by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.retrieve_multiple(ids=[1, 2, 3]) + + Get file_metadatas by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.retrieve_multiple(external_ids=["abc", "def"]) + """ + utils.assert_type(ids, "id", [List], allow_none=True) + utils.assert_type(external_ids, "external_id", [List], allow_none=True) + return self._retrieve_multiple(ids=ids, external_ids=external_ids, wrap_ids=True) + + def list( + self, + name: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + uploaded_time: Dict[str, Any] = None, + external_id_prefix: str = None, + uploaded: bool = None, + limit: int = 25, + ) -> FileMetadataList: + """List files + + Args: + name (str): Name of the file. + mime_type (str): File type. E.g. text/plain, application/pdf, .. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): Only include files that reference these specific asset IDs. + source (str): The source of this event. + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + uploaded_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project. + uploaded (bool): Whether or not the actual file is uploaded. This field is returned only by the API, it has no effect in a post body. + limit (int, optional): Max number of files to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + FileMetadataList: The requested files. + + Examples: + + List files metadata and filter on external id prefix:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> file_list = c.files.list(limit=5, external_id_prefix="prefix") + + Iterate over files metadata:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for file_metadata in c.files: + ... file_metadata # do something with the file metadata + + Iterate over chunks of files metadata to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for file_list in c.files(chunk_size=2500): + ... file_list # do something with the files + """ + filter = FileMetadataFilter( + name, + mime_type, + metadata, + asset_ids, + source, + created_time, + last_updated_time, + uploaded_time, + external_id_prefix, + uploaded, + ).dump(camel_case=True) + return self._list(method="POST", limit=limit, filter=filter) + + def delete(self, id: Union[int, List[int]] = None, external_id: Union[str, List[str]] = None) -> None: + """Delete files + + Args: + id (Union[int, List[int]]): Id or list of ids + external_id (Union[str, List[str]]): str or list of str + + Returns: + None + + Examples: + + Delete files by id or external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.delete(id=[1,2,3], external_id="3") + """ + self._delete_multiple(wrap_ids=True, ids=id, external_ids=external_id) + + def update( + self, item: Union[FileMetadata, FileMetadataUpdate, List[Union[FileMetadata, FileMetadataUpdate]]] + ) -> Union[FileMetadata, FileMetadataList]: + """Update files + + Args: + item (Union[FileMetadata, FileMetadataUpdate, List[Union[FileMetadata, FileMetadataUpdate]]]): file(s) to update. + + Returns: + Union[FileMetadata, FileMetadataList]: The updated files. + + Examples: + + Update file metadata that you have fetched. This will perform a full update of the file metadata:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> file_metadata = c.files.retrieve(id=1) + >>> file_metadata.description = "New description" + >>> res = c.files.update(file_metadata) + + Perform a partial update on file metadata, updating the source and adding a new field to metadata:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import FileMetadataUpdate + >>> c = CogniteClient() + >>> my_update = FileMetadataUpdate(id=1).source.set("new source").metadata.add({"key": "value"}) + >>> res = c.files.update(my_update) + """ + return self._update_multiple(cls=FileMetadataList, resource_path=self._RESOURCE_PATH, items=item) + + def search( + self, name: str = None, filter: Union[FileMetadataFilter, dict] = None, limit: int = None + ) -> FileMetadataList: + """Search for files. + + Args: + name (str, optional): Prefix and fuzzy search on name. + filter (Union[FileMetadataFilter, dict], optional): Filter to apply. Performs exact match on these fields. + limit (int, optional): Max number of results to return. + + Returns: + FileMetadataList: List of requested files metadata. + + Examples: + + Search for a file:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.search(name="some name") + """ + return self._search(search={"name": name}, filter=filter, limit=limit) + + def upload( + self, + path: str, + external_id: str = None, + name: str = None, + source: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + recursive: bool = False, + overwrite: bool = False, + ) -> Union[FileMetadata, FileMetadataList]: + """Upload a file + + Args: + path (str): Path to the file you wish to upload. If path is a directory, this method will upload all files in that directory. + external_id (str): External Id provided by client. Should be unique within the project. + name (str): No description. + source (str): The source of the file. + mime_type (str): File type. E.g. text/plain, application/pdf, ... + metadata (Dict[str, Any]): Customizable extra data about the file. String key -> String value. + asset_ids (List[int]): No description. + recursive (bool): If path is a directory, upload all contained files recursively. + overwrite (bool): If 'overwrite' is set to true, and the POST body content specifies a 'externalId' field, + fields for the file found for externalId can be overwritten. The default setting is false. + If metadata is included in the request body, all of the original metadata will be overwritten. + The actual file will be overwritten after successful upload. If there is no successful upload, the + current file contents will be kept. + File-Asset mappings only change if explicitly stated in the assetIds field of the POST json body. + Do not set assetIds in request body if you want to keep the current file-asset mappings. + + Returns: + Union[FileMetadata, FileMetadataList]: The file metadata of the uploaded file(s). + + Examples: + + Upload a file in a given path:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.upload("/path/to/file", name="my_file") + + If name is omitted, this method will use the name of the file + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.upload("/path/to/file") + + You can also upload all files in a directory by setting path to the path of a directory:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.upload("/path/to/my/directory") + + """ + file_metadata = FileMetadata( + name=name, + external_id=external_id, + source=source, + mime_type=mime_type, + metadata=metadata, + asset_ids=asset_ids, + ) + if os.path.isfile(path): + if not name: + file_metadata.name = os.path.basename(path) + return self._upload_file_from_path(file_metadata, path, overwrite) + elif os.path.isdir(path): + tasks = [] + if recursive: + for root, _, files in os.walk(path): + for file in files: + file_path = os.path.join(root, file) + basename = os.path.basename(file_path) + tasks.append((FileMetadata(name=basename), file_path, overwrite)) + else: + for file_name in os.listdir(path): + file_path = os.path.join(path, file_name) + if os.path.isfile(file_path): + tasks.append((FileMetadata(name=file_name), file_path, overwrite)) + tasks_summary = utils.execute_tasks_concurrently(self._upload_file_from_path, tasks, self._max_workers) + tasks_summary.raise_compound_exception_if_failed_tasks(task_unwrap_fn=lambda x: x[0].name) + return FileMetadataList(tasks_summary.results) + raise ValueError("path '{}' does not exist".format(path)) + + def _upload_file_from_path(self, file: FileMetadata, file_path: str, overwrite: bool): + with open(file_path, "rb") as f: + file_metadata = self.upload_bytes(f.read(), overwrite=overwrite, **file.dump(camel_case=True)) + return file_metadata + + def upload_bytes( + self, + content: Union[str, bytes], + external_id: str = None, + name: str = None, + source: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + overwrite: bool = False, + ): + """Upload bytes or string. + + Args: + content (Union[str, bytes]): The content to upload. + external_id (str): External Id provided by client. Should be unique within the project. + name (str): No description. + source (str): The source of the file. + mime_type (str): File type. E.g. text/plain, application/pdf,... + metadata (Dict[str, Any]): Customizable extra data about the file. String key -> String value. + asset_ids (List[int]): No description. + overwrite (bool): If 'overwrite' is set to true, and the POST body content specifies a 'externalId' field, + fields for the file found for externalId can be overwritten. The default setting is false. + If metadata is included in the request body, all of the original metadata will be overwritten. + The actual file will be overwritten after successful upload. If there is no successful upload, the + current file contents will be kept. + File-Asset mappings only change if explicitly stated in the assetIds field of the POST json body. + Do not set assetIds in request body if you want to keep the current file-asset mappings. + + Examples: + + Upload a file from memory:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.upload_bytes(b"some content", name="my_file", asset_ids=[1,2,3]) + + """ + file_metadata = FileMetadata( + name=name, + external_id=external_id, + source=source, + mime_type=mime_type, + metadata=metadata, + asset_ids=asset_ids, + ) + + res = self._post( + url_path=self._RESOURCE_PATH, json=file_metadata.dump(camel_case=True), params={"overwrite": overwrite} + ) + returned_file_metadata = res.json() + upload_url = returned_file_metadata.pop("uploadUrl") + headers = {"X-Upload-Content-Type": file_metadata.mime_type, "content-length": str(len(content))} + self._request_session.put(upload_url, data=content, headers=headers) + return FileMetadata._load(returned_file_metadata) + + def download( + self, directory: str, id: Union[int, List[int]] = None, external_id: Union[str, List[str]] = None + ) -> None: + """Download files by id or external id. + + This method will stream all files to disk, never keeping more than 2MB of a given file in memory. + + Args: + directory (str): Directory to download the file(s) to. + id (Union[int, List[int]], optional): Id or list of ids + external_id (Union[str, List[str]), optional): External ID or list of external ids. + + Returns: + None + + Examples: + + Download files by id and external id into directory 'my_directory':: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.files.download(directory="my_directory", id=[1,2,3], external_id=["abc", "def"]) + """ + all_ids = self._process_ids(ids=id, external_ids=external_id, wrap_ids=True) + id_to_metadata = self._get_id_to_metadata_map(all_ids) + self._download_files_to_directory(directory, all_ids, id_to_metadata) + + def _get_id_to_metadata_map(self, all_ids): + ids = [id["id"] for id in all_ids if "id" in id] + external_ids = [id["externalId"] for id in all_ids if "externalId" in id] + + files_metadata = self.retrieve_multiple(ids=ids, external_ids=external_ids) + + id_to_metadata = {} + for f in files_metadata: + id_to_metadata[f.id] = f + id_to_metadata[f.external_id] = f + + return id_to_metadata + + def _download_files_to_directory(self, directory, all_ids, id_to_metadata): + tasks = [(directory, id, id_to_metadata) for id in all_ids] + tasks_summary = utils.execute_tasks_concurrently( + self._process_file_download, tasks, max_workers=self._max_workers + ) + tasks_summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: id_to_metadata[utils.unwrap_identifer(task[1])], + str_format_element_fn=lambda metadata: metadata.id, + ) + + def _process_file_download(self, directory, identifier, id_to_metadata): + download_link = self._post(url_path="/files/downloadlink", json={"items": [identifier]}).json()["items"][0][ + "downloadUrl" + ] + id = utils.unwrap_identifer(identifier) + file_metadata = id_to_metadata[id] + file_path = os.path.join(directory, file_metadata.name) + self._download_file_to_path(download_link, file_path) + + def _download_file_to_path(self, download_link: str, path: str, chunk_size: int = 2 ** 21): + with self._request_session.get(download_link, stream=True) as r: + with open(path, "wb") as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + def download_bytes(self, id: int = None, external_id: str = None) -> bytes: + """Download a file as bytes. + + Args: + id (int, optional): Id of the file + external_id (str, optional): External id of the file + + Examples: + + Download a file's content into memory:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> file_content = c.files.download_bytes(id=1) + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + all_ids = self._process_ids(ids=id, external_ids=external_id, wrap_ids=True) + res = self._post(url_path="/files/downloadlink", json={"items": all_ids}) + dl_link = res.json()["items"][0]["downloadUrl"] + return self._download_file(dl_link) + + def _download_file(self, download_link: str) -> bytes: + res = self._request_session.get(download_link) + return res.content diff --git a/cognite/client/_api/iam.py b/cognite/client/_api/iam.py new file mode 100644 index 0000000000..8a1de18f54 --- /dev/null +++ b/cognite/client/_api/iam.py @@ -0,0 +1,227 @@ +import numbers +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import ( + APIKey, + APIKeyList, + Group, + GroupList, + SecurityCategory, + SecurityCategoryList, + ServiceAccount, + ServiceAccountList, +) +from cognite.client.utils import _utils as utils + + +class IAMAPI(APIClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.service_accounts = ServiceAccountsAPI(*args, **kwargs) + self.api_keys = APIKeysAPI(*args, **kwargs) + self.groups = GroupsAPI(*args, **kwargs) + self.security_categories = SecurityCategoriesAPI(*args, **kwargs) + + +class ServiceAccountsAPI(APIClient): + _RESOURCE_PATH = "/serviceaccounts" + _LIST_CLASS = ServiceAccountList + + def list(self) -> ServiceAccountList: + """List service accounts. + + Returns: + ServiceAccountList: List of service accounts. + """ + return ServiceAccountList._load(self._get(url_path=self._RESOURCE_PATH).json()["items"]) + + def create( + self, service_account: Union[ServiceAccount, List[ServiceAccount]] + ) -> Union[ServiceAccount, ServiceAccountList]: + """Create one or more new service accounts. + + Args: + service_account (Union[ServiceAccount, List[ServiceAccount]]): The service account(s) to create. + + Returns: + Union[ServiceAccount, ServiceAccountList]: The created service account(s). + """ + return self._create_multiple(items=service_account) + + def delete(self, id: Union[int, List[int]]) -> None: + """Delete one or more service accounts. + + Args: + id (Union[int, List[int]]): ID or list of IDs to delete. + + Returns: + None + """ + self._delete_multiple(ids=id, wrap_ids=False) + + +class APIKeysAPI(APIClient): + _RESOURCE_PATH = "/apikeys" + _LIST_CLASS = APIKeyList + + def list(self, include_deleted: bool = False, all: bool = False, service_account_id: bool = None) -> APIKeyList: + """List api keys. + + Args: + include_deleted (bool): Whether or not to include deleted api keys. Defaults to False. + all (bool): Whether or not to return all api keys for this project. Requires users:list acl. Defaults to False. + service_account_id (int): Get api keys for this service account only. Only available to admin users. + + Returns: + APIKeyList: List of api keys. + """ + res = self._get( + self._RESOURCE_PATH, + params={"all": all, "serviceAccountId": service_account_id, "includeDeleted": include_deleted}, + ) + return APIKeyList._load(res.json()["items"]) + + def create(self, service_account_id: Union[int, List[int]]) -> Union[APIKey, APIKeyList]: + """Create a new api key for one or more service accounts. + + Args: + service_account_id (Union[int, List[int]]): ID or list of IDs of service accounts to create an api key for. + + Returns: + Union[APIKey, APIKeyList]: API key or list of api keys. + """ + utils.assert_type(service_account_id, "service_account_id", [numbers.Integral, list]) + if isinstance(service_account_id, numbers.Integral): + items = {"serviceAccountId": service_account_id} + else: + items = [{"serviceAccountId": sa_id} for sa_id in service_account_id] + return self._create_multiple(items=items) + + def delete(self, id: Union[int, List[int]]) -> None: + """Delete one or more api keys. + + Args: + id (Union[int, List[int]]): ID or list of IDs of api keys to delete. + + Returns: + None + """ + self._delete_multiple(ids=id, wrap_ids=False) + + +class GroupsAPI(APIClient): + _RESOURCE_PATH = "/groups" + _LIST_CLASS = GroupList + + def list(self, all: bool = False) -> GroupList: + """List groups. + + Args: + all (bool): Whether to get all groups, only available with the groups:list acl. + + Returns: + GroupList: List of groups. + """ + res = self._get(self._RESOURCE_PATH, params={"all": all}) + return GroupList._load(res.json()["items"]) + + def create(self, group: Union[Group, List[Group]]) -> Union[Group, GroupList]: + """Create one or more groups. + + Args: + group (Union[Group, List[Group]]): Group or list of groups to create. + Returns: + Union[Group, GroupList]: The created group(s). + """ + return self._create_multiple(group) + + def delete(self, id: Union[int, List[int]]) -> None: + """Delete one or more groups. + + Args: + id (Union[int, List[int]]): ID or list of IDs of groups to delete. + + Returns: + None + """ + self._delete_multiple(ids=id, wrap_ids=False) + + def list_service_accounts(self, id: int) -> ServiceAccountList: + """List service accounts in a group. + + Args: + id (int): List service accounts which are a member of this group. + + Returns: + ServiceAccountList: List of service accounts. + """ + resource_path = self._RESOURCE_PATH + "/{}/serviceaccounts".format(id) + return ServiceAccountList._load(self._get(resource_path).json()["items"]) + + def add_service_account(self, id: int, service_account_id: Union[int, List[int]]) -> None: + """Add one or more service accounts to a group. + + Args: + id (int): Add service accounts to the group with this id. + service_account_id (Union[int, List[int]]): Add these service accounts to the specified group. + + Returns: + None + """ + resource_path = self._RESOURCE_PATH + "/{}/serviceaccounts".format(id) + self._create_multiple(cls=ServiceAccountList, resource_path=resource_path, items=service_account_id) + + def remove_service_account(self, id: int, service_account_id: Union[int, List[int]]) -> None: + """Remove one or more service accounts from a group. + + Args: + id (int): Remove service accounts from the group with this id. + service_account_id: Remove these service accounts from the specified group. + + Returns: + None + """ + url_path = self._RESOURCE_PATH + "/{}/serviceaccounts/remove".format(id) + all_ids = self._process_ids(service_account_id, None, False) + self._post(url_path, json={"items": all_ids}) + + +class SecurityCategoriesAPI(APIClient): + _RESOURCE_PATH = "/securitycategories" + _LIST_CLASS = SecurityCategoryList + + def list(self, limit: int = 25) -> SecurityCategoryList: + """List security categories. + + Args: + limit (int): Max number of security categories to return. Defaults to 25. + + Returns: + SecurityCategoryList: List of security categories + """ + return self._list(method="GET", limit=limit) + + def create( + self, security_category: Union[SecurityCategory, List[SecurityCategory]] + ) -> Union[SecurityCategory, SecurityCategoryList]: + """Create one or more security categories. + + Args: + group (Union[SecurityCategory, List[SecurityCategory]]): Security category or list of categories to create. + + Returns: + Union[SecurityCategory, SecurityCategoryList]: The created security category or categories. + """ + return self._create_multiple(security_category) + + def delete(self, id: Union[int, List[int]]) -> None: + """Delete one or more security categories. + + Args: + id (Union[int, List[int]]): ID or list of IDs of security categories to delete. + + Returns: + None + """ + self._delete_multiple(ids=id, wrap_ids=False) diff --git a/cognite/client/_api/login.py b/cognite/client/_api/login.py new file mode 100644 index 0000000000..2a0c2c6d3c --- /dev/null +++ b/cognite/client/_api/login.py @@ -0,0 +1,23 @@ +from cognite.client._api_client import APIClient +from cognite.client.data_classes.login import LoginStatus + + +class LoginAPI(APIClient): + _RESOURCE_PATH = "/login" + + def status(self) -> LoginStatus: + """Check login status + + Returns: + LoginStatus: The login status of the current api key. + + Examples: + Check the current login status and get the project:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> login_status = c.login.status() + >>> project = login_status.project + + """ + return LoginStatus._load(self._get(self._RESOURCE_PATH + "/status").json()) diff --git a/cognite/client/_api/raw.py b/cognite/client/_api/raw.py new file mode 100644 index 0000000000..b06f411d52 --- /dev/null +++ b/cognite/client/_api/raw.py @@ -0,0 +1,437 @@ +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import Database, DatabaseList, Row, RowList, Table, TableList +from cognite.client.utils import _utils as utils + + +class RawAPI(APIClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.databases = RawDatabasesAPI(*args, **kwargs) + self.tables = RawTablesAPI(*args, **kwargs) + self.rows = RawRowsAPI(*args, **kwargs) + + +class RawDatabasesAPI(APIClient): + _RESOURCE_PATH = "/raw/dbs" + _LIST_CLASS = DatabaseList + + def __call__(self, chunk_size: int = None) -> Generator[Union[Database, DatabaseList], None, None]: + """Iterate over databases + + Fetches dbs as they are iterated over, so you keep a limited number of dbs in memory. + + Args: + chunk_size (int, optional): Number of dbs to return in each chunk. Defaults to yielding one db a time. + """ + return self._list_generator(chunk_size=chunk_size, method="GET") + + def __iter__(self) -> Generator[Database, None, None]: + return self.__call__() + + def create(self, name: Union[str, List[str]]) -> Union[Database, DatabaseList]: + """Create one or more databases. + + Args: + name (Union[str, List[str]]): A db name or list of db names to create. + + Returns: + Union[Database, DatabaseList]: Database or list of databases that has been created. + + Examples: + + Create a new database:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.raw.databases.create("db1") + """ + utils.assert_type(name, "name", [str, list]) + if isinstance(name, str): + items = {"name": name} + else: + items = [{"name": n} for n in name] + return self._create_multiple(items=items) + + def delete(self, name: Union[str, List[str]]) -> None: + """Delete one or more databases. + + Args: + name (Union[str, List[str]]): A db name or list of db names to delete. + + Returns: + None + + Examples: + + Delete a list of databases:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.raw.databases.delete(["db1", "db2"]) + """ + utils.assert_type(name, "name", [str, list]) + if isinstance(name, str): + name = [name] + items = [{"name": n} for n in name] + chunks = utils.split_into_chunks(items, self._DELETE_LIMIT) + tasks = [{"url_path": self._RESOURCE_PATH + "/delete", "json": {"items": chunk}} for chunk in chunks] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], task_list_element_unwrap_fn=lambda el: el["name"] + ) + + def list(self, limit: int = 25) -> DatabaseList: + """List databases + + Args: + limit (int, optional): Maximum number of databases to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + DatabaseList: List of requested databases. + + Examples: + + List the first 5 databases:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> db_list = c.raw.databases.list(limit=5) + + Iterate over databases:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for db in c.raw.databases: + ... db # do something with the db + + Iterate over chunks of databases to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for db_list in c.raw.databases(chunk_size=2500): + ... db_list # do something with the dbs + """ + return self._list(method="GET", limit=limit) + + +class RawTablesAPI(APIClient): + _RESOURCE_PATH = "/raw/dbs/{}/tables" + _LIST_CLASS = TableList + + def __call__(self, db_name: str, chunk_size: int = None) -> Generator[Union[Table, TableList], None, None]: + """Iterate over tables + + Fetches tables as they are iterated over, so you keep a limited number of tables in memory. + + Args: + db_name (str): Name of the database to iterate over tables for + chunk_size (int, optional): Number of tables to return in each chunk. Defaults to yielding one table a time. + """ + for tb in self._list_generator( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name), + chunk_size=chunk_size, + method="GET", + ): + yield self._set_db_name_on_tables(tb, db_name) + + def create(self, db_name: str, name: Union[str, List[str]]) -> Union[Table, TableList]: + """Create one or more tables. + + Args: + db_name (str): Database to create the tables in. + name (Union[str, List[str]]): A table name or list of table names to create. + + Returns: + Union[Table, TableList]: Table or list of tables that has been created. + + Examples: + + Create a new table in a database:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.raw.tables.create("db1", "table1") + """ + utils.assert_type(name, "name", [str, list]) + if isinstance(name, str): + items = {"name": name} + else: + items = [{"name": n} for n in name] + tb = self._create_multiple( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name), items=items + ) + return self._set_db_name_on_tables(tb, db_name) + + def delete(self, db_name: str, name: Union[str, List[str]]) -> None: + """Delete one or more tables. + + Args: + db_name (str): Database to delete tables from. + name (Union[str, List[str]]): A table name or list of table names to delete. + + Returns: + None + + Examples: + + Delete a list of tables:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.raw.tables.delete("db1", ["table1", "table2"]) + """ + utils.assert_type(name, "name", [str, list]) + if isinstance(name, str): + name = [name] + items = [{"name": n} for n in name] + chunks = utils.split_into_chunks(items, self._DELETE_LIMIT) + tasks = [ + { + "url_path": utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name) + "/delete", + "json": {"items": chunk}, + } + for chunk in chunks + ] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], task_list_element_unwrap_fn=lambda el: el["name"] + ) + + def list(self, db_name: str, limit: int = 25) -> TableList: + """List tables + + Args: + db_name (str): The database to list tables from. + limit (int, optional): Maximum number of tables to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + TableList: List of requested tables. + + Examples: + + List the first 5 tables:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> table_list = c.raw.tables.list("db1", limit=5) + + Iterate over tables:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for table in c.raw.tables(db_name="db1"): + ... table # do something with the table + + Iterate over chunks of tables to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for table_list in c.raw.tables(db_name="db1", chunk_size=2500): + ... table_list # do something with the tables + """ + tb = self._list( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name), method="GET", limit=limit + ) + return self._set_db_name_on_tables(tb, db_name) + + def _set_db_name_on_tables(self, tb: Union[Table, TableList], db_name: str) -> Union[Table, TableList]: + if isinstance(tb, Table): + tb._db_name = db_name + return tb + elif isinstance(tb, TableList): + for t in tb: + t._db_name = db_name + return tb + raise TypeError("tb must be Table or TableList") + + +class RawRowsAPI(APIClient): + _RESOURCE_PATH = "/raw/dbs/{}/tables/{}/rows" + _LIST_CLASS = RowList + + def __call__( + self, db_name: str, table_name: str, chunk_size: int = None + ) -> Generator[Union[Row, RowList], None, None]: + """Iterate over rows. + + Fetches rows as they are iterated over, so you keep a limited number of rows in memory. + + Args: + db_name (str): Name of the database + table_name (str): Name of the table to iterate over rows for + chunk_size (int, optional): Number of rows to return in each chunk. Defaults to yielding one row a time. + """ + return self._list_generator( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name, table_name), + chunk_size=chunk_size, + method="GET", + ) + + def insert( + self, db_name: str, table_name: str, row: Union[List[Row], Row, Dict], ensure_parent: bool = False + ) -> None: + """Insert one or more rows into a table. + + Args: + db_name (str): Name of the database. + table_name (str): Name of the table. + row (Union[List[Row], Row, Dict]): The row(s) to insert + ensure_parent (bool): Create database/table if they don't already exist. + + Returns: + None + + Examples: + + Insert new rows into a table:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> rows = {"r1": {"col1": "val1", "col2": "val1"}, "r2": {"col1": "val2", "col2": "val2"}} + >>> res = c.raw.rows.insert("db1", "table1", rows) + """ + chunks = self._process_row_input(row) + + tasks = [ + { + "url_path": utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name, table_name), + "json": {"items": chunk}, + "params": {"ensureParent": ensure_parent}, + } + for chunk in chunks + ] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], task_list_element_unwrap_fn=lambda row: row["key"] + ) + + def _process_row_input(self, row: List[Union[List, Dict, Row]]): + utils.assert_type(row, "row", [list, dict, Row]) + rows = [] + if isinstance(row, dict): + for key, columns in row.items(): + rows.append({"key": key, "columns": columns}) + elif isinstance(row, list): + for elem in row: + if isinstance(elem, Row): + rows.append(elem.dump(camel_case=True)) + else: + raise TypeError("list elements must be Row objects.") + elif isinstance(row, Row): + rows.append(row.dump(camel_case=True)) + return utils.split_into_chunks(rows, self._CREATE_LIMIT) + + def delete(self, db_name: str, table_name: str, key: Union[str, List[str]]) -> None: + """Delete rows from a table. + + Args: + db_name (str): Name of the database. + table_name (str): Name of the table. + key (Union[str, List[str]]): The key(s) of the row(s) to delete. + + Returns: + None + + Examples: + + Delete rows from table:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> keys_to_delete = ["k1", "k2", "k3"] + >>> res = c.raw.rows.delete("db1", "table1", keys_to_delete) + """ + utils.assert_type(key, "key", [str, list]) + if isinstance(key, str): + key = [key] + items = [{"key": k} for k in key] + chunks = utils.split_into_chunks(items, self._DELETE_LIMIT) + tasks = [ + ( + { + "url_path": utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name, table_name) + "/delete", + "json": {"items": chunk}, + } + ) + for chunk in chunks + ] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], task_list_element_unwrap_fn=lambda el: el["key"] + ) + + def retrieve(self, db_name: str, table_name: str, key: str) -> Optional[Row]: + """Retrieve a single row by key. + + Args: + db_name (str): Name of the database. + table_name (str): Name of the table. + key (str): The key of the row to retrieve. + + Returns: + Optional[Row]: The requested row. + + Examples: + + Retrieve a row with key 'k1' from tablew 't1' in database 'db1':: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> row = c.raw.rows.retrieve("db1", "t1", "k1") + """ + return self._retrieve( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name, table_name), id=key + ) + + def list( + self, + db_name: str, + table_name: str, + min_last_updated_time: int = None, + max_last_updated_time: int = None, + limit: int = 25, + ) -> RowList: + """List rows in a table. + + Args: + db_name (str): Name of the database. + table_name (str): Name of the table. + min_last_updated_time (int): Rows must have been last updated after this time. ms since epoch. + max_last_updated_time (int): Rows must have been last updated before this time. ms since epoch. + limit (int): The number of rows to retrieve. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + RowList: The requested rows. + + Examples: + + List rows:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> row_list = c.raw.rows.list("db1", "t1", limit=5) + + Iterate over rows:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for row in c.raw.rows(db_name="db1", table_name="t1"): + ... row # do something with the row + + Iterate over chunks of rows to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for row_list in c.raw.rows(db_name="db1", table_name="t1", chunk_size=2500): + ... row_list # do something with the rows + """ + return self._list( + resource_path=utils.interpolate_and_url_encode(self._RESOURCE_PATH, db_name, table_name), + limit=limit, + method="GET", + filter={"minLastUpdatedTime": min_last_updated_time, "maxLastUpdatedTime": max_last_updated_time}, + ) diff --git a/cognite/client/_api/three_d.py b/cognite/client/_api/three_d.py new file mode 100644 index 0000000000..46af5c4c3e --- /dev/null +++ b/cognite/client/_api/three_d.py @@ -0,0 +1,447 @@ +import json +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import ( + ThreeDAssetMapping, + ThreeDAssetMappingList, + ThreeDModel, + ThreeDModelList, + ThreeDModelRevision, + ThreeDModelRevisionList, + ThreeDModelRevisionUpdate, + ThreeDModelUpdate, + ThreeDNodeList, + ThreeDRevealNodeList, + ThreeDRevealRevision, + ThreeDRevealSectorList, +) +from cognite.client.utils import _utils as utils + + +class ThreeDAPI(APIClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.models = ThreeDModelsAPI(*args, **kwargs) + self.revisions = ThreeDRevisionsAPI(*args, **kwargs) + self.files = ThreeDFilesAPI(*args, **kwargs) + self.asset_mappings = ThreeDAssetMappingAPI(*args, **kwargs) + # self.reveal = ThreeDRevealAPI(*args, **kwargs) + + +class ThreeDModelsAPI(APIClient): + _RESOURCE_PATH = "/3d/models" + _LIST_CLASS = ThreeDModelList + + def __call__( + self, chunk_size: int = None, published: bool = False + ) -> Generator[Union[ThreeDModel, ThreeDModelList], None, None]: + """Iterate over 3d models + + Fetches 3d models as they are iterated over, so you keep a limited number of 3d models in memory. + + Args: + chunk_size (int, optional): Number of 3d models to return in each chunk. Defaults to yielding one model a time. + published (bool): Filter based on whether or not the model has published revisions. + + Yields: + Union[ThreeDModel, ThreeDModelList]: yields ThreeDModel one by one if chunk is not specified, else ThreeDModelList objects. + """ + return self._list_generator(method="GET", chunk_size=chunk_size, filter={"published": published}) + + def __iter__(self) -> Generator[ThreeDModel, None, None]: + """Iterate over 3d models + + Fetches models as they are iterated over, so you keep a limited number of models in memory. + + Yields: + ThreeDModel: yields models one by one. + """ + return self.__call__() + + def retrieve(self, id: int) -> ThreeDModel: + """Retrieve a 3d model by id + + Args: + id (int): Get the model with this id. + + Returns: + ThreeDModel: The requested 3d model. + """ + return self._retrieve(id) + + def list(self, published: bool = False, limit: int = 25) -> ThreeDModelList: + """List 3d models. + + Args: + published (bool): Filter based on whether or not the model has published revisions. + limit (int): Maximum number of models to retrieve. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ThreeDModelList: The list of 3d models. + """ + return self._list(method="GET", filter={"published": published}, limit=limit) + + def create(self, name: Union[str, List[str]]) -> Union[ThreeDModel, ThreeDModelList]: + """Create new 3d models. + + Args: + name (Union[str, List[str]): The name of the 3d model(s) to create. + + Returns: + Union[ThreeDModel, ThreeDModelList]: The created 3d model(s). + """ + utils.assert_type(name, "name", [str, list]) + if isinstance(name, str): + name = {"name": name} + else: + name = [{"name": n} for n in name] + return self._create_multiple(items=name) + + def update( + self, item: Union[ThreeDModel, ThreeDModelUpdate, List[Union[ThreeDModel, ThreeDModelList]]] + ) -> Union[ThreeDModel, ThreeDModelList]: + """Update 3d models. + + Args: + item (Union[ThreeDModel, ThreeDModelUpdate, List[Union[ThreeDModel, ThreeDModelUpdate]]]): ThreeDModel(s) to update + + Returns: + Union[ThreeDModel, ThreeDModelList]: Updated ThreeDModel(s) + """ + return self._update_multiple(items=item) + + def delete(self, id: Union[int, List[int]]) -> None: + """Delete 3d models. + + Args: + id (Union[int, List[int]]): ID or list of IDs to delete. + + Returns: + None + """ + self._delete_multiple(ids=id, wrap_ids=True) + + +class ThreeDRevisionsAPI(APIClient): + _RESOURCE_PATH = "/3d/models/{}/revisions" + _LIST_CLASS = ThreeDModelRevisionList + + def __call__( + self, model_id: int, chunk_size: int = None, published: bool = False + ) -> Generator[Union[ThreeDModelRevision, ThreeDModelRevisionList], None, None]: + """Iterate over 3d model revisions + + Fetches 3d model revisions as they are iterated over, so you keep a limited number of 3d model revisions in memory. + + Args: + model_id (int): Iterate over revisions for the model with this id. + chunk_size (int, optional): Number of 3d model revisions to return in each chunk. Defaults to yielding one model a time. + published (bool): Filter based on whether or not the revision has been published. + + Yields: + Union[ThreeDModelRevision, ThreeDModelRevisionList]: yields ThreeDModelRevision one by one if chunk is not + specified, else ThreeDModelRevisionList objects. + """ + return self._list_generator( + resource_path=self._RESOURCE_PATH.format(model_id), + method="GET", + chunk_size=chunk_size, + filter={"published": published}, + ) + + def retrieve(self, model_id: int, id: int) -> ThreeDModelRevision: + """Retrieve a 3d model revision by id + + Args: + model_id (int): Get the revision under the model with this id. + id (int): Get the model revision with this id. + + Returns: + ThreeDModelRevision: The requested 3d model revision. + """ + return self._retrieve(resource_path=self._RESOURCE_PATH.format(model_id), id=id) + + def create( + self, model_id: int, revision: Union[ThreeDModelRevision, List[ThreeDModelRevision]] + ) -> Union[ThreeDModelRevision, ThreeDModelRevisionList]: + """Create a revisions for a specified 3d model. + + Args: + model_id (int): Create revisions for this model. + revision (Union[ThreeDModelRevision, List[ThreeDModelRevision]]): The revision(s) to create. + + Returns: + Union[ThreeDModelRevision, ThreeDModelRevisionList]: The created revision(s) + """ + return self._create_multiple(resource_path=self._RESOURCE_PATH.format(model_id), items=revision) + + def list(self, model_id: int, published: bool = False, limit: int = 25) -> ThreeDModelRevisionList: + """List 3d model revisions. + + Args: + model_id (int): List revisions under the model with this id. + published (bool): Filter based on whether or not the revision is published. + limit (int): Maximum number of models to retrieve. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ThreeDModelRevisionList: The list of 3d model revisions. + """ + return self._list( + resource_path=self._RESOURCE_PATH.format(model_id), + method="GET", + filter={"published": published}, + limit=limit, + ) + + def update( + self, + model_id: int, + item: Union[ + ThreeDModelRevision, ThreeDModelRevisionUpdate, List[Union[ThreeDModelRevision, ThreeDModelRevisionList]] + ], + ) -> Union[ThreeDModelRevision, ThreeDModelRevisionList]: + """Update 3d model revisions. + + Args: + model_id (int): Update the revision under the model with this id. + item (Union[ThreeDModelRevision, ThreeDModelRevisionUpdate, List[Union[ThreeDModelRevision, ThreeDModelRevisionUpdate]]]): + ThreeDModelRevision(s) to update + + Returns: + Union[ThreeDModelRevision, ThreeDModelRevisionList]: Updated ThreeDModelRevision(s) + """ + return self._update_multiple(resource_path=self._RESOURCE_PATH.format(model_id), items=item) + + def delete(self, model_id: int, id: Union[int, List[int]]) -> None: + """Delete 3d model revisions. + + Args: + model_id (int): Delete the revision under the model with this id. + id (Union[int, List[int]]): ID or list of IDs to delete. + + Returns: + None + """ + self._delete_multiple(resource_path=self._RESOURCE_PATH.format(model_id), ids=id, wrap_ids=True) + + def update_thumbnail(self, model_id: int, revision_id: int, file_id: int) -> None: + """Update a revision thumbnail. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + file_id (int): Id of the thumbnail file in the Files API. + + Returns: + None + """ + resource_path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}/thumbnail", model_id, revision_id) + body = {"fileId": file_id} + self._post(resource_path, json=body) + + def list_nodes( + self, model_id: int, revision_id: int, node_id: int = None, depth: int = None, limit: int = 25 + ) -> ThreeDNodeList: + """Retrieves a list of nodes from the hierarchy in the 3D Model. + + You can also request a specific subtree with the 'nodeId' query parameter and limit the depth of + the resulting subtree with the 'depth' query parameter. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + node_id (int): ID of the root node of the subtree you request (default is the root node). + depth (int): Get sub nodes up to this many levels below the specified node. Depth 0 is the root node. + limit (int): Maximun number of nodes to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ThreeDNodeList: The list of 3d nodes. + """ + resource_path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}/nodes", model_id, revision_id) + return self._list( + cls=ThreeDNodeList, + resource_path=resource_path, + method="GET", + limit=limit, + filter={"depth": depth, "nodeId": node_id}, + ) + + def list_ancestor_nodes( + self, model_id: int, revision_id: int, node_id: int = None, limit: int = 25 + ) -> ThreeDNodeList: + """Retrieves a list of ancestor nodes of a given node, including itself, in the hierarchy of the 3D model + + You can also request a specific subtree with the 'nodeId' query parameter and limit the depth of + the resulting subtree with the 'depth' query parameter. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + node_id (int): ID of the node to get the ancestors of. + depth (int): Get sub nodes up to this many levels below the specified node. Depth 0 is the root node. + limit (int): Maximun number of nodes to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ThreeDNodeList: The list of 3d nodes. + """ + resource_path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}/nodes", model_id, revision_id) + return self._list( + cls=ThreeDNodeList, resource_path=resource_path, method="GET", limit=limit, filter={"nodeId": node_id} + ) + + +class ThreeDFilesAPI(APIClient): + _RESOURCE_PATH = "/3d/files" + + def retrieve(self, id: int) -> bytes: + """Retrieve the contents of a 3d file by id. + + Args: + id (int): The id of the file to retrieve. + + Returns: + bytes: The contents of the file. + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}", id) + return self._get(path).content + + +class ThreeDAssetMappingAPI(APIClient): + _RESOURCE_PATH = "/3d/models/{}/revisions/{}/mappings" + _LIST_CLASS = ThreeDAssetMappingList + + def list( + self, model_id: int, revision_id: int, node_id: int = None, asset_id: int = None, limit: int = 25 + ) -> ThreeDAssetMappingList: + """List 3D node asset mappings. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + node_id (int): List only asset mappings associated with this node. + asset_id (int): List only asset mappings associated with this asset. + limit (int): Maximum number of asset mappings to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + ThreeDAssetMappingList: The list of asset mappings. + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH, model_id, revision_id) + return self._list( + resource_path=path, method="GET", filter={"nodeId": node_id, "assetId": asset_id}, limit=limit + ) + + def create( + self, model_id: int, revision_id: int, asset_mapping: Union[ThreeDAssetMapping, List[ThreeDAssetMapping]] + ) -> Union[ThreeDAssetMapping, ThreeDAssetMappingList]: + """Create 3d node asset mappings. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + asset_mapping (Union[ThreeDAssetMapping, List[ThreeDAssetMapping]]): The asset mapping(s) to create. + + Returns: + Union[ThreeDAssetMapping, ThreeDAssetMappingList]: The created asset mapping(s). + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH, model_id, revision_id) + return self._create_multiple(resource_path=path, items=asset_mapping) + + def delete( + self, model_id: int, revision_id: int, asset_mapping: Union[ThreeDAssetMapping, List[ThreeDAssetMapping]] + ) -> None: + """Delete 3d node asset mappings. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + asset_mapping (Union[ThreeDAssetMapping, List[ThreeDAssetMapping]]): The asset mapping(s) to delete. + + Returns: + None + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH, model_id, revision_id) + utils.assert_type(asset_mapping, "asset_mapping", [list, ThreeDAssetMapping]) + if isinstance(asset_mapping, ThreeDAssetMapping): + asset_mapping = [asset_mapping] + chunks = utils.split_into_chunks([a.dump(camel_case=True) for a in asset_mapping], self._DELETE_LIMIT) + tasks = [{"url_path": path + "/delete", "json": {"items": chunk}} for chunk in chunks] + summary = utils.execute_tasks_concurrently(self._post, tasks, self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], + task_list_element_unwrap_fn=lambda el: ThreeDAssetMapping._load(el), + str_format_element_fn=lambda el: (el.asset_id, el.node_id), + ) + + +class ThreeDRevealAPI(APIClient): + _RESOURCE_PATH = "/3d/reveal/models/{}/revisions" + + def retrieve_revision(self, model_id: int, revision_id: int): + """Retrieve a revision. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH, model_id) + return self._retrieve(cls=ThreeDRevealRevision, resource_path=path, id=revision_id) + + def list_nodes(self, model_id: int, revision_id: int, depth: int = None, node_id: int = None, limit: int = 25): + """List 3D nodes. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + depth (int, optional): Get sub nodes up to this many levels below the specified node. + node_id (int, optional): ID of the root note of the subtree you request. + limit (int, optional): Maximun number of nodes to retrieve. Defaults to 25. Set to -1, float("inf") or None + to return all items. + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}/nodes", model_id, revision_id) + return self._list( + cls=ThreeDRevealNodeList, + resource_path=path, + method="GET", + filter={"depth": depth, "nodeId": node_id}, + limit=limit, + ) + + def list_ancestor_nodes(self, model_id: int, revision_id: int, node_id: int, limit: int = 25): + """Retrieve a revision. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + node_id (int): ID of the node to get the ancestors of. + limit (int, optional): Maximun number of nodes to retrieve. Defaults to 25. Set to -1, float("inf") or None + to return all items. + """ + path = utils.interpolate_and_url_encode( + self._RESOURCE_PATH + "/{}/nodes/{}/ancestors", model_id, revision_id, node_id + ) + return self._list(cls=ThreeDRevealNodeList, resource_path=path, method="GET", limit=limit) + + def list_sectors(self, model_id: int, revision_id: int, bounding_box: Dict[str, List] = None, limit: int = 25): + """Retrieve a revision. + + Args: + model_id (int): Id of the model. + revision_id (int): Id of the revision. + bounding_box (Dict[str, List], optional): Bounding box to restrict search to. If given, only return sectors that intersect the given bounding box. + limit (int, optional): Maximum number of items to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + """ + path = utils.interpolate_and_url_encode(self._RESOURCE_PATH + "/{}/sectors", model_id, revision_id) + return self._list( + cls=ThreeDRevealSectorList, + resource_path=path, + method="GET", + filter={"boundingBox": json.dumps(bounding_box)}, + limit=limit, + ) diff --git a/cognite/client/_api/time_series.py b/cognite/client/_api/time_series.py new file mode 100644 index 0000000000..62db7fd4ce --- /dev/null +++ b/cognite/client/_api/time_series.py @@ -0,0 +1,241 @@ +import numbers +from typing import * + +from cognite.client._api_client import APIClient +from cognite.client.data_classes import TimeSeries, TimeSeriesFilter, TimeSeriesList, TimeSeriesUpdate +from cognite.client.utils import _utils as utils + + +class TimeSeriesAPI(APIClient): + _RESOURCE_PATH = "/timeseries" + _LIST_CLASS = TimeSeriesList + + def __call__( + self, chunk_size: int = None, include_metadata: bool = False, asset_ids: List[int] = None + ) -> Generator[Union[TimeSeries, TimeSeriesList], None, None]: + """Iterate over time series + + Fetches time series as they are iterated over, so you keep a limited number of objects in memory. + + Args: + chunk_size (int, optional): Number of time series to return in each chunk. Defaults to yielding one event a time. + include_metadata (bool, optional): Whether or not to include metadata + asset_id (int, optional): List time series related to this asset. + + Yields: + Union[TimeSeries, TimeSeriesList]: yields TimeSeries one by one if chunk is not specified, else TimeSeriesList objects. + """ + filter = {"includeMetadata": include_metadata, "assetIds": str(asset_ids) if asset_ids else None} + return self._list_generator(method="GET", chunk_size=chunk_size, filter=filter) + + def __iter__(self) -> Generator[TimeSeries, None, None]: + """Iterate over time series + + Fetches time series as they are iterated over, so you keep a limited number of metadata objects in memory. + + Yields: + TimeSeries: yields TimeSeries one by one. + """ + return self.__call__() + + def retrieve(self, id: Optional[int] = None, external_id: Optional[str] = None) -> Optional[TimeSeries]: + """Retrieve a single time series by id. + + Args: + id (int, optional): ID + external_id (str, optional): External ID + + Returns: + Optional[TimeSeries]: Requested time series or None if it does not exist. + + Examples: + + Get time series by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.retrieve(id=1) + + Get time series by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.retrieve(external_id="1") + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + return self._retrieve_multiple(ids=id, external_ids=external_id, wrap_ids=True) + + def retrieve_multiple( + self, ids: Optional[List[int]] = None, external_ids: Optional[List[str]] = None + ) -> TimeSeriesList: + """Retrieve multiple time series by id. + + Args: + ids (List[int], optional): IDs + external_ids (List[str], optional): External IDs + + Returns: + TimeSeriesList: The requested time series. + + Examples: + + Get time series by id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.retrieve_multiple(ids=[1, 2, 3]) + + Get time series by external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.retrieve_multiple(external_ids=["abc", "def"]) + """ + utils.assert_type(ids, "id", [List], allow_none=True) + utils.assert_type(external_ids, "external_id", [List], allow_none=True) + return self._retrieve_multiple(ids=ids, external_ids=external_ids, wrap_ids=True) + + def list( + self, include_metadata: bool = False, asset_ids: Optional[List[int]] = None, limit: int = 25 + ) -> TimeSeriesList: + """Iterate over time series + + Fetches time series as they are iterated over, so you keep a limited number of objects in memory. + + Args: + include_metadata (bool, optional): Whether or not to include metadata + asset_ids (List[int], optional): List time series related to these assets. + limit (int, optional): Max number of time series to return. Defaults to 25. Set to -1, float("inf") or None + to return all items. + + Returns: + TimeSeriesList: The requested time series. + + Examples: + + List time series:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.list(limit=5) + + Iterate over time series:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for ts in c.time_series: + ... ts # do something with the time_series + + Iterate over chunks of time series to reduce memory load:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> for ts_list in c.time_series(chunk_size=2500): + ... ts_list # do something with the time_series + """ + filter = {"includeMetadata": include_metadata, "assetIds": str(asset_ids) if asset_ids else None} + return self._list(method="GET", filter=filter, limit=limit) + + def create(self, time_series: Union[TimeSeries, List[TimeSeries]]) -> Union[TimeSeries, TimeSeriesList]: + """Create one or more time series. + + Args: + time_series (Union[TimeSeries, List[TimeSeries]]): TimeSeries or list of TimeSeries to create. + + Returns: + Union[TimeSeries, TimeSeriesList]: The created time series. + + Examples: + + Create a new time series:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import TimeSeries + >>> c = CogniteClient() + >>> ts = c.time_series.create(TimeSeries(name="my ts")) + """ + return self._create_multiple(items=time_series) + + def delete(self, id: Union[int, List[int]] = None, external_id: Union[str, List[str]] = None) -> None: + """Delete one or more time series. + + Args: + id (Union[int, List[int]): Id or list of ids + external_id (Union[str, List[str]]): External ID or list of external ids + + Returns: + None + + Examples: + + Delete time series by id or external id:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.delete(id=[1,2,3], external_id="3") + """ + self._delete_multiple(wrap_ids=True, ids=id, external_ids=external_id) + + def update( + self, item: Union[TimeSeries, TimeSeriesUpdate, List[Union[TimeSeries, TimeSeriesUpdate]]] + ) -> Union[TimeSeries, TimeSeriesList]: + """Update one or more time series. + + Args: + item (Union[TimeSeries, TimeSeriesUpdate, List[Union[TimeSeries, TimeSeriesUpdate]]]): Time series to update + + Returns: + Union[TimeSeries, TimeSeriesList]: Updated time series. + + Examples: + + Update a time series that you have fetched. This will perform a full update of the time series:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.retrieve(id=1) + >>> res.description = "New description" + >>> res = c.time_series.update(res) + + Perform a partial update on a time series, updating the description and adding a new field to metadata:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import TimeSeriesUpdate + >>> c = CogniteClient() + >>> my_update = TimeSeriesUpdate(id=1).description.set("New description").metadata.add({"key": "value"}) + >>> res = c.time_series.update(my_update) + """ + return self._update_multiple(items=item) + + def search( + self, + name: str = None, + description: str = None, + query: str = None, + filter: Union[TimeSeriesFilter, Dict] = None, + limit: int = None, + ) -> TimeSeriesList: + """Search for time series. + + Args: + name (str, optional): Prefix and fuzzy search on name. + description (str, optional): Prefix and fuzzy search on description. + query (str, optional): Search on name and description using wildcard search on each of the words (separated + by spaces). Retrieves results where at least one word must match. Example: 'some other' + filter (Union[TimeSeriesFilter, Dict], optional): Filter to apply. Performs exact match on these fields. + limit (int, optional): Max number of results to return. + + Returns: + TimeSeriesList: List of requested time series. + + Examples: + + Search for a time series:: + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> res = c.time_series.search(name="some name") + """ + return self._search( + search={"name": name, "description": description, "query": query}, filter=filter, limit=limit + ) diff --git a/cognite/client/_api_client.py b/cognite/client/_api_client.py index c761719c2d..d8daf736ad 100644 --- a/cognite/client/_api_client.py +++ b/cognite/client/_api_client.py @@ -1,306 +1,577 @@ -import functools import gzip -import json +import json as _json import logging import os import re -from typing import Any, Dict +from typing import Any, Dict, List, Union -import numpy +import requests.utils from requests import Response, Session from requests.adapters import HTTPAdapter +from requests.structures import CaseInsensitiveDict from urllib3 import Retry -from cognite.client.exceptions import APIError +from cognite.client.data_classes._base import CogniteFilter, CogniteResource, CogniteUpdate +from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError +from cognite.client.utils import _utils as utils log = logging.getLogger("cognite-sdk") -DEFAULT_NUM_OF_RETRIES = 5 -HTTP_METHODS_TO_RETRY = [429, 500, 502, 503] +BACKOFF_MAX = 30 +DEFAULT_MAX_POOL_SIZE = 50 +DEFAULT_MAX_RETRIES = 10 + + +class RetryWithMaxBackoff(Retry): + def get_backoff_time(self): + return min(BACKOFF_MAX, super().get_backoff_time()) + + +def _get_status_codes_to_retry(): + env_codes = os.getenv("COGNITE_STATUS_FORCELIST") + if env_codes is None: + return [429, 500, 502, 503] + return [int(c) for c in env_codes.split(",")] def _init_requests_session(): session = Session() - num_of_retries = int(os.getenv("COGNITE_NUM_RETRIES", DEFAULT_NUM_OF_RETRIES)) - retry = Retry( - total=num_of_retries, - read=num_of_retries, - connect=num_of_retries, - backoff_factor=0.5, - status_forcelist=HTTP_METHODS_TO_RETRY, - raise_on_status=False, + session_with_retry = Session() + num_of_retries = int(os.getenv("COGNITE_MAX_RETRIES", DEFAULT_MAX_RETRIES)) + max_pool_size = int(os.getenv("COGNITE_MAX_CONNECTION_POOL_SIZE", DEFAULT_MAX_POOL_SIZE)) + adapter = HTTPAdapter( + max_retries=RetryWithMaxBackoff( + total=num_of_retries, connect=num_of_retries, read=0, status=0, backoff_factor=0.5, raise_on_status=False + ), + pool_maxsize=max_pool_size, + ) + adapter_with_retry = HTTPAdapter( + max_retries=RetryWithMaxBackoff( + total=num_of_retries, + backoff_factor=0.5, + status_forcelist=_get_status_codes_to_retry(), + method_whitelist=False, + raise_on_status=False, + ), + pool_maxsize=max_pool_size, ) - adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) - return session - - -_REQUESTS_SESSION = _init_requests_session() - - -def _status_is_valid(status_code: int): - return status_code < 400 - - -def _raise_API_error(res: Response): - x_request_id = res.headers.get("X-Request-Id") - code = res.status_code - extra = {} - try: - error = res.json()["error"] - if isinstance(error, str): - msg = error - else: - msg = error["message"] - extra = error.get("extra") - except: - msg = res.content - - log.error("HTTP Error %s: %s", code, msg, extra={"X-Request-ID": x_request_id, "extra": extra}) - raise APIError(msg, code, x_request_id, extra=extra) - - -def _log_request(res: Response, **kwargs): - method = res.request.method - url = res.request.url - status_code = res.status_code - - extra = kwargs.copy() - extra["headers"] = res.request.headers - if "api-key" in extra.get("headers", {}): - extra["headers"]["api-key"] = None - - http_protocol_version = ".".join(list(str(res.raw.version))) - - log.info("HTTP/{} {} {} {}".format(http_protocol_version, method, url, status_code), extra=extra) - - -def request_method(method=None): - @functools.wraps(method) - def wrapper(client_instance, url, *args, **kwargs): - if not url.startswith("/"): - raise ValueError("URL must start with '/'") - full_url = client_instance._base_url + url - - # Hack to allow running model hosting requests against local emulator - if os.getenv("USE_MODEL_HOSTING_EMULATOR") == "1": - full_url = _model_hosting_emulator_url_converter(full_url) + session_with_retry.mount("http://", adapter_with_retry) + session_with_retry.mount("https://", adapter_with_retry) + return session, session_with_retry - default_headers = client_instance._headers.copy() - default_headers.update(kwargs.get("headers") or {}) - kwargs["headers"] = default_headers - res = method(client_instance, full_url, *args, **kwargs) - if _status_is_valid(res.status_code): - return res - _raise_API_error(res) - return wrapper - - -def _model_hosting_emulator_url_converter(url): - pattern = "https://api.cognitedata.com/api/0.6/projects/(.*)/analytics/models(.*)" - res = re.match(pattern, url) - if res is not None: - project = res.group(1) - path = res.group(2) - return "http://localhost:8000/api/0.1/projects/{}/models{}".format(project, path) - return url +_REQUESTS_SESSION, _REQUESTS_SESSION_WITH_RETRY = _init_requests_session() class APIClient: - _LIMIT = 1000 + _RESOURCE_PATH = None + _LIST_CLASS = None def __init__( self, version: str = None, project: str = None, + api_key: str = None, base_url: str = None, - num_of_workers: int = None, - cookies: Dict = None, + max_workers: int = None, headers: Dict = None, timeout: int = None, + cognite_client=None, ): self._request_session = _REQUESTS_SESSION + self._request_session_with_retry = _REQUESTS_SESSION_WITH_RETRY self._project = project + self._api_key = api_key __base_path = "/api/{}/projects/{}".format(version, project) if version else "" self._base_url = base_url + __base_path - self._num_of_workers = num_of_workers - self._cookies = cookies - self._headers = headers + self._max_workers = max_workers + self._headers = self._configure_headers(headers) self._timeout = timeout + self._cognite_client = cognite_client - @request_method - def _delete(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): - res = self._request_session.delete( - url, params=params, headers=headers, cookies=self._cookies, timeout=self._timeout - ) - _log_request(res) - return res - - def _autopaged_get(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): - params = params.copy() - items = [] - while True: - url = re.sub("{}(/.*)".format(self._base_url), r"\1", url) - res = self._get(url, params=params, headers=headers) - params["cursor"] = res.json()["data"].get("nextCursor") - items.extend(res.json()["data"]["items"]) - next_cursor = res.json()["data"].get("nextCursor") - if not next_cursor: - break - res._content = json.dumps({"data": {"items": items}}).encode() - return res - - @request_method - def _get(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None, autopaging: bool = False): - if autopaging: - return self._autopaged_get(url, params, headers) - res = self._request_session.get( - url, params=params, headers=headers, cookies=self._cookies, timeout=self._timeout - ) - _log_request(res) - return res - - @request_method - def _post(self, url: str, body: Dict[str, Any], params: Dict[str, Any] = None, headers: Dict[str, Any] = None): - data = json.dumps(body, default=self._json_dumps_default) - if not os.getenv("COGNITE_DISABLE_GZIP", False): - headers["Content-Encoding"] = "gzip" - data = gzip.compress(data.encode()) - res = self._request_session.post( - url, data=data, headers=headers, params=params, cookies=self._cookies, timeout=self._timeout - ) - _log_request(res, body=body) - return res - - @request_method - def _put(self, url: str, body: Dict[str, Any] = None, headers: Dict[str, Any] = None): - data = json.dumps(body or {}, default=self._json_dumps_default) - if not os.getenv("COGNITE_DISABLE_GZIP", False): - headers["Content-Encoding"] = "gzip" - data = gzip.compress(data.encode()) - res = self._request_session.put(url, data=data, headers=headers, cookies=self._cookies, timeout=self._timeout) - _log_request(res, body=body) - return res - - @staticmethod - def _json_dumps_default(x): - if isinstance(x, numpy.int_): - return int(x) - if isinstance(x, numpy.float_): - return float(x) - if isinstance(x, numpy.bool_): - return bool(x) - return x.__dict__ - - -class CogniteResponse: - """Cognite Response class - - All responses inherit from this class. + self._CREATE_LIMIT = 1000 + self._LIST_LIMIT = 1000 + self._RETRIEVE_LIMIT = 1000 + self._DELETE_LIMIT = 1000 + self._UPDATE_LIMIT = 1000 - Examples: - All responses are pretty-printable:: + def _delete(self, url_path: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + return self._do_request("DELETE", url_path, params=params, headers=headers, timeout=self._timeout) - from cognite.client import CogniteClient + def _get(self, url_path: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + return self._do_request("GET", url_path, params=params, headers=headers, timeout=self._timeout) - client = CogniteClient() - res = client.assets.get_assets(limit=1) + def _post(self, url_path: str, json: Dict[str, Any], params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + return self._do_request("POST", url_path, json=json, headers=headers, params=params, timeout=self._timeout) - print(res) + def _put(self, url_path: str, json: Dict[str, Any] = None, headers: Dict[str, Any] = None): + return self._do_request("PUT", url_path, json=json, headers=headers, timeout=self._timeout) - All endpoints which support paging have an ``autopaging`` flag which may be set to true in order to sequentially - fetch all resources. If for some reason, you want to do this manually, you may use the next_cursor() method on - the response object. Here is an example of that:: + def _do_request(self, method: str, url_path: str, **kwargs): + is_retryable, full_url = self._resolve_url(method, url_path) - from cognite.client import CogniteClient + json_payload = kwargs.get("json") + headers = self._headers.copy() + headers.update(kwargs.get("headers") or {}) - client = CogniteClient() + if json_payload: + data = _json.dumps(json_payload, default=utils.json_dump_default) + kwargs["data"] = data + if method in ["PUT", "POST"] and not os.getenv("COGNITE_DISABLE_GZIP", False): + kwargs["data"] = gzip.compress(data.encode()) + headers["Content-Encoding"] = "gzip" - asset_list = [] + kwargs["headers"] = headers - cursor = None - while True: - res = client.assets.get_assets(cursor=cursor) - asset_list.extend(res.to_json()) - cursor = res.next_cursor() - if cursor is None: - break - - print(asset_list) - """ - - def __init__(self, internal_representation): - self.internal_representation = internal_representation + if is_retryable: + res = self._request_session_with_retry.request(method=method, url=full_url, **kwargs) + else: + res = self._request_session.request(method=method, url=full_url, **kwargs) - def __str__(self): - return json.dumps(self.to_json(), indent=4, sort_keys=True) + if not self._status_is_valid(res.status_code): + self._raise_API_error(res, payload=json_payload) + self._log_request(res, payload=json_payload) + return res - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"][0] + def _configure_headers(self, additional_headers): + headers = CaseInsensitiveDict() + headers.update(requests.utils.default_headers()) + headers["api-key"] = self._api_key + headers["content-type"] = "application/json" + headers["accept"] = "application/json" + headers["x-cdp-sdk"] = "CognitePythonSDK:{}".format(utils.get_current_sdk_version()) + if "User-Agent" in headers: + headers["User-Agent"] += " " + utils.get_user_agent() + else: + headers["User-Agent"] = utils.get_user_agent() + headers.update(additional_headers) + return headers + + def _resolve_url(self, method: str, url_path: str): + if not url_path.startswith("/"): + raise ValueError("URL path must start with '/'") + full_url = self._base_url + url_path + is_retryable = self._is_retryable(method, full_url) + # Hack to allow running model hosting requests against local emulator + full_url = self._apply_model_hosting_emulator_url_filter(full_url) + return is_retryable, full_url + + def _is_retryable(self, method, path): + valid_methods = ["GET", "POST", "PUT", "DELETE"] + match = re.match("(?:http|https)://[a-z\d.:]+(?:/api/v1/projects/[^/]+)?(/.+)", path) + + if not match: + raise ValueError("Path {} is not valid. Cannot resolve whether or not it is retryable".format(path)) + if method not in valid_methods: + raise ValueError("Method {} is not valid. Must be one of {}".format(method, valid_methods)) + path_end = match.group(1) + # TODO: This following set should be generated from the openapi spec somehow. + retryable_post_endpoints = { + "/assets/list", + "/assets/byids", + "/assets/search", + "/events/list", + "/events/byids", + "/events/search", + "/files/list", + "/files/byids", + "/files/search", + "/files/initupload", + "/files/downloadlink", + "/timeseries/byids", + "/timeseries/search", + "/timeseries/data", + "/timeseries/data/list", + "/timeseries/data/latest", + "/timeseries/data/delete", + } + if method == "GET": + return True + if method == "POST" and path_end in retryable_post_endpoints: + return True + return False + + def _retrieve( + self, id: Union[int, str], cls=None, resource_path: str = None, params: Dict = None, headers: Dict = None + ): + cls = cls or self._LIST_CLASS._RESOURCE + resource_path = resource_path or self._RESOURCE_PATH + try: + res = self._get( + url_path=utils.interpolate_and_url_encode(resource_path + "/{}", str(id)), + params=params, + headers=headers, + ) + return cls._load(res.json(), cognite_client=self._cognite_client) + except CogniteAPIError as e: + if e.code != 404: + raise + + def _retrieve_multiple( + self, + wrap_ids: bool, + cls=None, + resource_path: str = None, + ids: Union[List[int], int] = None, + external_ids: Union[List[str], str] = None, + headers: Dict = None, + ): + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + all_ids = self._process_ids(ids, external_ids, wrap_ids=wrap_ids) + id_chunks = utils.split_into_chunks(all_ids, self._RETRIEVE_LIMIT) + + tasks = [ + {"url_path": resource_path + "/byids", "json": {"items": id_chunk}, "headers": headers} + for id_chunk in id_chunks + ] + tasks_summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + + if tasks_summary.exceptions: + e = tasks_summary.exceptions[0] + if isinstance(e, CogniteAPIError) and e.code == 400 and e.missing is not None: + if self._is_single_identifier(ids, external_ids): + return None + raise CogniteNotFoundError(e.missing) from e + raise tasks_summary.exceptions[0] + + retrieved_items = tasks_summary.joined_results(lambda res: res.json()["items"]) + + if self._is_single_identifier(ids, external_ids): + return cls._RESOURCE._load(retrieved_items[0], cognite_client=self._cognite_client) + return cls._load(retrieved_items, cognite_client=self._cognite_client) + + def _list_generator( + self, + method: str, + cls=None, + resource_path: str = None, + limit: int = None, + chunk_size: int = None, + filter: Dict = None, + headers: Dict = None, + ): + if limit == -1 or limit == float("inf"): + limit = None + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + total_items_retrieved = 0 + current_limit = self._LIST_LIMIT + if chunk_size and chunk_size <= self._LIST_LIMIT: + current_limit = chunk_size + next_cursor = None + filter = filter or {} + current_items = [] + while True: + if limit: + num_of_remaining_items = limit - total_items_retrieved + if num_of_remaining_items < self._LIST_LIMIT: + current_limit = num_of_remaining_items + + if method == "GET": + params = filter.copy() + params["limit"] = current_limit + params["cursor"] = next_cursor + res = self._get(url_path=resource_path, params=params, headers=headers) + elif method == "POST": + body = {"filter": filter, "limit": current_limit, "cursor": next_cursor} + res = self._post(url_path=resource_path + "/list", json=body, headers=headers) + else: + raise ValueError("_list_generator parameter `method` must be GET or POST, not %s", method) + last_received_items = res.json()["items"] + current_items.extend(last_received_items) + + if not chunk_size: + for item in current_items: + yield cls._RESOURCE._load(item, cognite_client=self._cognite_client) + total_items_retrieved += len(current_items) + current_items = [] + elif len(current_items) >= chunk_size or len(last_received_items) < self._LIST_LIMIT: + items_to_yield = current_items[:chunk_size] + yield cls._load(items_to_yield, cognite_client=self._cognite_client) + total_items_retrieved += len(items_to_yield) + current_items = current_items[chunk_size:] + + next_cursor = res.json().get("nextCursor") + if total_items_retrieved == limit or next_cursor is None: + break - def next_cursor(self): - """Returns next cursor to use for paging through results. Returns ``None`` if there are no more results.""" - if self.internal_representation.get("data"): - return self.internal_representation.get("data").get("nextCursor") + def _list( + self, + method: str, + cls=None, + resource_path: str = None, + limit: int = None, + filter: Dict = None, + headers: Dict = None, + ): + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + items = [] + for resource_list in self._list_generator( + cls=cls, + resource_path=resource_path, + method=method, + limit=limit, + chunk_size=self._LIST_LIMIT, + filter=filter, + headers=headers, + ): + items.extend(resource_list.data) + return cls(items, cognite_client=self._cognite_client) + + def _create_multiple( + self, + items: Union[List[Any], Any], + cls: Any = None, + resource_path: str = None, + params: Dict = None, + headers: Dict = None, + limit=None, + ): + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + limit = limit or self._CREATE_LIMIT + single_item = not isinstance(items, list) + if single_item: + items = [items] + + items_split = [] + for i in range(0, len(items), limit): + if isinstance(items[i], CogniteResource): + items_chunk = [item.dump(camel_case=True) for item in items[i : i + limit]] + else: + items_chunk = [item for item in items[i : i + limit]] + items_split.append({"items": items_chunk}) + + tasks = [(resource_path, task_items, params, headers) for task_items in items_split] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + + def unwrap_element(el): + if isinstance(el, dict): + return cls._RESOURCE._load(el) + else: + return el + + def str_format_element(el): + if isinstance(el, CogniteResource): + dumped = el.dump() + if "external_id" in dumped: + return dumped["external_id"] + return dumped + return el + + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task[1]["items"], + task_list_element_unwrap_fn=unwrap_element, + str_format_element_fn=str_format_element, + ) + created_resources = summary.joined_results(lambda res: res.json()["items"]) - def previous_cursor(self): - """Returns previous cursor to use for paging through results. Returns ``None`` if there are no more results.""" - if self.internal_representation.get("data"): - return self.internal_representation.get("data").get("previousCursor") + if single_item: + return cls._RESOURCE._load(created_resources[0], cognite_client=self._cognite_client) + return cls._load(created_resources, cognite_client=self._cognite_client) + def _delete_multiple( + self, + wrap_ids: bool, + resource_path: str = None, + ids: Union[List[int], int] = None, + external_ids: Union[List[str], str] = None, + params: Dict = None, + headers: Dict = None, + ): + resource_path = resource_path or self._RESOURCE_PATH + all_ids = self._process_ids(ids, external_ids, wrap_ids) + id_chunks = utils.split_into_chunks(all_ids, self._DELETE_LIMIT) + tasks = [ + {"url_path": resource_path + "/delete", "json": {"items": chunk}, "params": params, "headers": headers} + for chunk in id_chunks + ] + summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], task_list_element_unwrap_fn=lambda el: el + ) -class CogniteCollectionResponse(CogniteResponse): - """Cognite Collection Response class + def _update_multiple( + self, + items: Union[List[Any], Any], + cls: Any = None, + resource_path: str = None, + params: Dict = None, + headers: Dict = None, + ): + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + patch_objects = [] + single_item = not isinstance(items, list) + if single_item: + items = [items] + + for item in items: + if isinstance(item, CogniteResource): + patch_objects.append(self._convert_resource_to_patch_object(item, cls._UPDATE._get_update_properties())) + elif isinstance(item, CogniteUpdate): + patch_objects.append(item.dump()) + else: + raise ValueError("update item must be of type CogniteResource or CogniteUpdate") + patch_object_chunks = utils.split_into_chunks(patch_objects, self._UPDATE_LIMIT) + + tasks = [ + {"url_path": resource_path + "/update", "json": {"items": chunk}, "params": params, "headers": headers} + for chunk in patch_object_chunks + ] + + tasks_summary = utils.execute_tasks_concurrently(self._post, tasks, max_workers=self._max_workers) + tasks_summary.raise_compound_exception_if_failed_tasks( + task_unwrap_fn=lambda task: task["json"]["items"], + task_list_element_unwrap_fn=lambda el: utils.unwrap_identifer(el), + ) + updated_items = tasks_summary.joined_results(lambda res: res.json()["items"]) - All collection responses inherit from this class. Collection responses are subscriptable and iterable. - """ + if single_item: + return cls._RESOURCE._load(updated_items[0], cognite_client=self._cognite_client) + return cls._load(updated_items, cognite_client=self._cognite_client) - _RESPONSE_CLASS = None + def _search( + self, + search: Dict, + filter: Union[Dict, CogniteFilter], + limit: int, + cls: Any = None, + resource_path: str = None, + params: Dict = None, + headers: Dict = None, + ): + utils.assert_type(filter, "filter", [dict, CogniteFilter], allow_none=True) + if isinstance(filter, CogniteFilter): + filter = filter.dump(camel_case=True) + elif isinstance(filter, dict): + filter = utils.convert_all_keys_to_camel_case(filter) + cls = cls or self._LIST_CLASS + resource_path = resource_path or self._RESOURCE_PATH + res = self._post( + url_path=resource_path + "/search", + json={"search": search, "filter": filter, "limit": limit}, + params=params, + headers=headers, + ) + return cls._load(res.json()["items"], cognite_client=self._cognite_client) - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"] + @staticmethod + def _convert_resource_to_patch_object(resource, update_attributes): + dumped_resource = resource.dump(camel_case=True) + has_id = "id" in dumped_resource + has_external_id = "externalId" in dumped_resource + utils.assert_exactly_one_of_id_or_external_id(dumped_resource.get("id"), dumped_resource.get("externalId")) + + patch_object = {"update": {}} + if has_id: + patch_object["id"] = dumped_resource.pop("id") + elif has_external_id: + patch_object["externalId"] = dumped_resource.pop("externalId") + + for key, value in dumped_resource.items(): + if key in update_attributes: + patch_object["update"][key] = {"set": value} + return patch_object - def __getitem__(self, index): - if isinstance(index, slice): - return self.__class__({"data": {"items": self.to_json()[index]}}) - return self._RESPONSE_CLASS({"data": {"items": [self.to_json()[index]]}}) + @staticmethod + def _process_ids( + ids: Union[List[int], int, None], external_ids: Union[List[str], str, None], wrap_ids: bool + ) -> List: + if external_ids is None and ids is None: + raise ValueError("No ids specified") + if external_ids and not wrap_ids: + raise ValueError("externalIds must be wrapped") + + if isinstance(ids, int): + ids = [ids] + elif isinstance(ids, list) or ids is None: + ids = ids or [] + else: + raise TypeError("ids must be int or list of int") - def __len__(self): - return len(self.to_json()) + if isinstance(external_ids, str): + external_ids = [external_ids] + elif isinstance(external_ids, list) or external_ids is None: + external_ids = external_ids or [] + else: + raise TypeError("external_ids must be str or list of str") - def __iter__(self): - self.counter = 0 - return self + if wrap_ids: + ids = [{"id": id} for id in ids] + external_ids = [{"externalId": external_id} for external_id in external_ids] - def __next__(self): - if self.counter > len(self.to_json()) - 1: - raise StopIteration - else: - self.counter += 1 - return self._RESPONSE_CLASS({"data": {"items": [self.to_json()[self.counter - 1]]}}) + all_ids = ids + external_ids + return all_ids -class CogniteResource: @staticmethod - def _to_camel_case(snake_case_string: str): - components = snake_case_string.split("_") - return components[0] + "".join(x.title() for x in components[1:]) - - def camel_case_dict(self): - new_d = {} - for key in self.__dict__: - new_d[self._to_camel_case(key)] = self.__dict__[key] - return new_d + def _is_single_identifier(ids, external_ids): + single_id = isinstance(ids, int) and external_ids is None + single_external_id = isinstance(external_ids, str) and ids is None + return single_id or single_external_id - def to_json(self): - return json.loads(json.dumps(self, default=lambda x: x.__dict__)) + @staticmethod + def _status_is_valid(status_code: int): + return status_code < 400 - def __eq__(self, other): - return type(self) == type(other) and self.to_json() == other.to_json() + @staticmethod + def _raise_API_error(res: Response, payload: Dict): + x_request_id = res.headers.get("X-Request-Id") + code = res.status_code + missing = None + duplicated = None + try: + error = res.json()["error"] + if isinstance(error, str): + msg = error + elif isinstance(error, Dict): + msg = error["message"] + missing = error.get("missing") + duplicated = error.get("duplicated") + else: + msg = res.content + except: + msg = res.content + + error_details = {"X-Request-ID": x_request_id} + if payload: + error_details["payload"] = payload + if missing: + error_details["missing"] = missing + if duplicated: + error_details["duplicated"] = duplicated + + log.debug("HTTP Error %s %s %s: %s", code, res.request.method, res.request.url, msg, extra=error_details) + raise CogniteAPIError(msg, code, x_request_id, missing=missing, duplicated=duplicated) - def __str__(self): - return json.dumps(self.__dict__, default=lambda x: x.__dict__, indent=4, sort_keys=True) + @staticmethod + def _log_request(res: Response, **kwargs): + method = res.request.method + url = res.request.url + status_code = res.status_code + + extra = kwargs.copy() + extra["headers"] = res.request.headers.copy() + if "api-key" in extra.get("headers", {}): + extra["headers"]["api-key"] = None + if extra["payload"] is None: + del extra["payload"] + + http_protocol_version = ".".join(list(str(res.raw.version))) + + log.info("HTTP/{} {} {} {}".format(http_protocol_version, method, url, status_code), extra=extra) + + def _apply_model_hosting_emulator_url_filter(self, full_url): + mlh_emul_url = os.getenv("MODEL_HOSTING_EMULATOR_URL") + if mlh_emul_url is not None: + pattern = "{}/analytics/models(.*)".format(self._base_url) + res = re.match(pattern, full_url) + if res is not None: + path = res.group(1) + return "{}/projects/{}/models{}".format(mlh_emul_url, self._project, path) + return full_url diff --git a/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v1_pb2.py b/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v1_pb2.py deleted file mode 100644 index d56a41b130..0000000000 --- a/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v1_pb2.py +++ /dev/null @@ -1,505 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: api_timeseries_data_v1.proto - -import sys - -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pb2 -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="api_timeseries_data_v1.proto", - package="api", - syntax="proto3", - serialized_pb=_b( - '\n\x1c\x61pi_timeseries_data_v1.proto\x12\x03\x61pi"3\n\x0fStringDatapoint\x12\x11\n\ttimestamp\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\t"4\n\x10NumericDatapoint\x12\x11\n\ttimestamp\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x01"<\n\x14StringTimeseriesData\x12$\n\x06points\x18\x01 \x03(\x0b\x32\x14.api.StringDatapoint">\n\x15NumericTimeseriesData\x12%\n\x06points\x18\x01 \x03(\x0b\x32\x15.api.NumericDatapoint"|\n\x0eTimeseriesData\x12/\n\nstringData\x18\x01 \x01(\x0b\x32\x19.api.StringTimeseriesDataH\x00\x12\x31\n\x0bnumericData\x18\x02 \x01(\x0b\x32\x1a.api.NumericTimeseriesDataH\x00\x42\x06\n\x04\x64\x61ta"\x8e\x01\n\x11TagTimeseriesData\x12\r\n\x05tagId\x18\x01 \x01(\t\x12/\n\nstringData\x18\x02 \x01(\x0b\x32\x19.api.StringTimeseriesDataH\x00\x12\x31\n\x0bnumericData\x18\x03 \x01(\x0b\x32\x1a.api.NumericTimeseriesDataH\x00\x42\x06\n\x04\x64\x61ta"K\n\x16MultiTagTimeseriesData\x12\x31\n\x11tagTimeseriesData\x18\x01 \x03(\x0b\x32\x16.api.TagTimeseriesDataB4\n\x17\x63om.cognite.data.api.v1P\x01\xb8\x01\x01\xaa\x02\x13\x43ognite.Data.Api.V1b\x06proto3' - ), -) - - -_STRINGDATAPOINT = _descriptor.Descriptor( - name="StringDatapoint", - full_name="api.StringDatapoint", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="timestamp", - full_name="api.StringDatapoint.timestamp", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="value", - full_name="api.StringDatapoint.value", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=37, - serialized_end=88, -) - - -_NUMERICDATAPOINT = _descriptor.Descriptor( - name="NumericDatapoint", - full_name="api.NumericDatapoint", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="timestamp", - full_name="api.NumericDatapoint.timestamp", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="value", - full_name="api.NumericDatapoint.value", - index=1, - number=2, - type=1, - cpp_type=5, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=90, - serialized_end=142, -) - - -_STRINGTIMESERIESDATA = _descriptor.Descriptor( - name="StringTimeseriesData", - full_name="api.StringTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="points", - full_name="api.StringTimeseriesData.points", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=144, - serialized_end=204, -) - - -_NUMERICTIMESERIESDATA = _descriptor.Descriptor( - name="NumericTimeseriesData", - full_name="api.NumericTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="points", - full_name="api.NumericTimeseriesData.points", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=206, - serialized_end=268, -) - - -_TIMESERIESDATA = _descriptor.Descriptor( - name="TimeseriesData", - full_name="api.TimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="stringData", - full_name="api.TimeseriesData.stringData", - index=0, - number=1, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="numericData", - full_name="api.TimeseriesData.numericData", - index=1, - number=2, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name="data", full_name="api.TimeseriesData.data", index=0, containing_type=None, fields=[] - ) - ], - serialized_start=270, - serialized_end=394, -) - - -_TAGTIMESERIESDATA = _descriptor.Descriptor( - name="TagTimeseriesData", - full_name="api.TagTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="tagId", - full_name="api.TagTimeseriesData.tagId", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="stringData", - full_name="api.TagTimeseriesData.stringData", - index=1, - number=2, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="numericData", - full_name="api.TagTimeseriesData.numericData", - index=2, - number=3, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name="data", full_name="api.TagTimeseriesData.data", index=0, containing_type=None, fields=[] - ) - ], - serialized_start=397, - serialized_end=539, -) - - -_MULTITAGTIMESERIESDATA = _descriptor.Descriptor( - name="MultiTagTimeseriesData", - full_name="api.MultiTagTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="tagTimeseriesData", - full_name="api.MultiTagTimeseriesData.tagTimeseriesData", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=541, - serialized_end=616, -) - -_STRINGTIMESERIESDATA.fields_by_name["points"].message_type = _STRINGDATAPOINT -_NUMERICTIMESERIESDATA.fields_by_name["points"].message_type = _NUMERICDATAPOINT -_TIMESERIESDATA.fields_by_name["stringData"].message_type = _STRINGTIMESERIESDATA -_TIMESERIESDATA.fields_by_name["numericData"].message_type = _NUMERICTIMESERIESDATA -_TIMESERIESDATA.oneofs_by_name["data"].fields.append(_TIMESERIESDATA.fields_by_name["stringData"]) -_TIMESERIESDATA.fields_by_name["stringData"].containing_oneof = _TIMESERIESDATA.oneofs_by_name["data"] -_TIMESERIESDATA.oneofs_by_name["data"].fields.append(_TIMESERIESDATA.fields_by_name["numericData"]) -_TIMESERIESDATA.fields_by_name["numericData"].containing_oneof = _TIMESERIESDATA.oneofs_by_name["data"] -_TAGTIMESERIESDATA.fields_by_name["stringData"].message_type = _STRINGTIMESERIESDATA -_TAGTIMESERIESDATA.fields_by_name["numericData"].message_type = _NUMERICTIMESERIESDATA -_TAGTIMESERIESDATA.oneofs_by_name["data"].fields.append(_TAGTIMESERIESDATA.fields_by_name["stringData"]) -_TAGTIMESERIESDATA.fields_by_name["stringData"].containing_oneof = _TAGTIMESERIESDATA.oneofs_by_name["data"] -_TAGTIMESERIESDATA.oneofs_by_name["data"].fields.append(_TAGTIMESERIESDATA.fields_by_name["numericData"]) -_TAGTIMESERIESDATA.fields_by_name["numericData"].containing_oneof = _TAGTIMESERIESDATA.oneofs_by_name["data"] -_MULTITAGTIMESERIESDATA.fields_by_name["tagTimeseriesData"].message_type = _TAGTIMESERIESDATA -DESCRIPTOR.message_types_by_name["StringDatapoint"] = _STRINGDATAPOINT -DESCRIPTOR.message_types_by_name["NumericDatapoint"] = _NUMERICDATAPOINT -DESCRIPTOR.message_types_by_name["StringTimeseriesData"] = _STRINGTIMESERIESDATA -DESCRIPTOR.message_types_by_name["NumericTimeseriesData"] = _NUMERICTIMESERIESDATA -DESCRIPTOR.message_types_by_name["TimeseriesData"] = _TIMESERIESDATA -DESCRIPTOR.message_types_by_name["TagTimeseriesData"] = _TAGTIMESERIESDATA -DESCRIPTOR.message_types_by_name["MultiTagTimeseriesData"] = _MULTITAGTIMESERIESDATA -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -StringDatapoint = _reflection.GeneratedProtocolMessageType( - "StringDatapoint", - (_message.Message,), - dict( - DESCRIPTOR=_STRINGDATAPOINT, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.StringDatapoint) - ), -) -_sym_db.RegisterMessage(StringDatapoint) - -NumericDatapoint = _reflection.GeneratedProtocolMessageType( - "NumericDatapoint", - (_message.Message,), - dict( - DESCRIPTOR=_NUMERICDATAPOINT, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.NumericDatapoint) - ), -) -_sym_db.RegisterMessage(NumericDatapoint) - -StringTimeseriesData = _reflection.GeneratedProtocolMessageType( - "StringTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_STRINGTIMESERIESDATA, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.StringTimeseriesData) - ), -) -_sym_db.RegisterMessage(StringTimeseriesData) - -NumericTimeseriesData = _reflection.GeneratedProtocolMessageType( - "NumericTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_NUMERICTIMESERIESDATA, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.NumericTimeseriesData) - ), -) -_sym_db.RegisterMessage(NumericTimeseriesData) - -TimeseriesData = _reflection.GeneratedProtocolMessageType( - "TimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_TIMESERIESDATA, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.TimeseriesData) - ), -) -_sym_db.RegisterMessage(TimeseriesData) - -TagTimeseriesData = _reflection.GeneratedProtocolMessageType( - "TagTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_TAGTIMESERIESDATA, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.TagTimeseriesData) - ), -) -_sym_db.RegisterMessage(TagTimeseriesData) - -MultiTagTimeseriesData = _reflection.GeneratedProtocolMessageType( - "MultiTagTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_MULTITAGTIMESERIESDATA, - __module__="api_timeseries_data_v1_pb2" - # @@protoc_insertion_point(class_scope:api.MultiTagTimeseriesData) - ), -) -_sym_db.RegisterMessage(MultiTagTimeseriesData) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions( - descriptor_pb2.FileOptions(), _b("\n\027com.cognite.data.api.v1P\001\270\001\001\252\002\023Cognite.Data.Api.V1") -) -# @@protoc_insertion_point(module_scope) diff --git a/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v2_pb2.py b/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v2_pb2.py deleted file mode 100644 index 8ba01efc46..0000000000 --- a/cognite/client/_auxiliary/_protobuf_descriptors/_api_timeseries_data_v2_pb2.py +++ /dev/null @@ -1,505 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: api_timeseries_data_v2.proto - -import sys - -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pb2 -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="api_timeseries_data_v2.proto", - package="api.v2", - syntax="proto3", - serialized_pb=_b( - '\n\x1c\x61pi_timeseries_data_v2.proto\x12\x06\x61pi.v2"3\n\x0fStringDatapoint\x12\x11\n\ttimestamp\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\t"4\n\x10NumericDatapoint\x12\x11\n\ttimestamp\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x01"?\n\x14StringTimeseriesData\x12\'\n\x06points\x18\x01 \x03(\x0b\x32\x17.api.v2.StringDatapoint"A\n\x15NumericTimeseriesData\x12(\n\x06points\x18\x01 \x03(\x0b\x32\x18.api.v2.NumericDatapoint"\x82\x01\n\x0eTimeseriesData\x12\x32\n\nstringData\x18\x01 \x01(\x0b\x32\x1c.api.v2.StringTimeseriesDataH\x00\x12\x34\n\x0bnumericData\x18\x02 \x01(\x0b\x32\x1d.api.v2.NumericTimeseriesDataH\x00\x42\x06\n\x04\x64\x61ta"\x95\x01\n\x13NamedTimeseriesData\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x32\n\nstringData\x18\x02 \x01(\x0b\x32\x1c.api.v2.StringTimeseriesDataH\x00\x12\x34\n\x0bnumericData\x18\x03 \x01(\x0b\x32\x1d.api.v2.NumericTimeseriesDataH\x00\x42\x06\n\x04\x64\x61ta"T\n\x18MultiNamedTimeseriesData\x12\x38\n\x13namedTimeseriesData\x18\x01 \x03(\x0b\x32\x1b.api.v2.NamedTimeseriesDataB1\n\x17\x63om.cognite.data.api.v2P\x01\xaa\x02\x13\x43ognite.Data.Api.V2b\x06proto3' - ), -) - - -_STRINGDATAPOINT = _descriptor.Descriptor( - name="StringDatapoint", - full_name="api.v2.StringDatapoint", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="timestamp", - full_name="api.v2.StringDatapoint.timestamp", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="value", - full_name="api.v2.StringDatapoint.value", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=40, - serialized_end=91, -) - - -_NUMERICDATAPOINT = _descriptor.Descriptor( - name="NumericDatapoint", - full_name="api.v2.NumericDatapoint", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="timestamp", - full_name="api.v2.NumericDatapoint.timestamp", - index=0, - number=1, - type=3, - cpp_type=2, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="value", - full_name="api.v2.NumericDatapoint.value", - index=1, - number=2, - type=1, - cpp_type=5, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=93, - serialized_end=145, -) - - -_STRINGTIMESERIESDATA = _descriptor.Descriptor( - name="StringTimeseriesData", - full_name="api.v2.StringTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="points", - full_name="api.v2.StringTimeseriesData.points", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=147, - serialized_end=210, -) - - -_NUMERICTIMESERIESDATA = _descriptor.Descriptor( - name="NumericTimeseriesData", - full_name="api.v2.NumericTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="points", - full_name="api.v2.NumericTimeseriesData.points", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=212, - serialized_end=277, -) - - -_TIMESERIESDATA = _descriptor.Descriptor( - name="TimeseriesData", - full_name="api.v2.TimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="stringData", - full_name="api.v2.TimeseriesData.stringData", - index=0, - number=1, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="numericData", - full_name="api.v2.TimeseriesData.numericData", - index=1, - number=2, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name="data", full_name="api.v2.TimeseriesData.data", index=0, containing_type=None, fields=[] - ) - ], - serialized_start=280, - serialized_end=410, -) - - -_NAMEDTIMESERIESDATA = _descriptor.Descriptor( - name="NamedTimeseriesData", - full_name="api.v2.NamedTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="name", - full_name="api.v2.NamedTimeseriesData.name", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="stringData", - full_name="api.v2.NamedTimeseriesData.stringData", - index=1, - number=2, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - _descriptor.FieldDescriptor( - name="numericData", - full_name="api.v2.NamedTimeseriesData.numericData", - index=2, - number=3, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name="data", full_name="api.v2.NamedTimeseriesData.data", index=0, containing_type=None, fields=[] - ) - ], - serialized_start=413, - serialized_end=562, -) - - -_MULTINAMEDTIMESERIESDATA = _descriptor.Descriptor( - name="MultiNamedTimeseriesData", - full_name="api.v2.MultiNamedTimeseriesData", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="namedTimeseriesData", - full_name="api.v2.MultiNamedTimeseriesData.namedTimeseriesData", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - file=DESCRIPTOR, - ) - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto3", - extension_ranges=[], - oneofs=[], - serialized_start=564, - serialized_end=648, -) - -_STRINGTIMESERIESDATA.fields_by_name["points"].message_type = _STRINGDATAPOINT -_NUMERICTIMESERIESDATA.fields_by_name["points"].message_type = _NUMERICDATAPOINT -_TIMESERIESDATA.fields_by_name["stringData"].message_type = _STRINGTIMESERIESDATA -_TIMESERIESDATA.fields_by_name["numericData"].message_type = _NUMERICTIMESERIESDATA -_TIMESERIESDATA.oneofs_by_name["data"].fields.append(_TIMESERIESDATA.fields_by_name["stringData"]) -_TIMESERIESDATA.fields_by_name["stringData"].containing_oneof = _TIMESERIESDATA.oneofs_by_name["data"] -_TIMESERIESDATA.oneofs_by_name["data"].fields.append(_TIMESERIESDATA.fields_by_name["numericData"]) -_TIMESERIESDATA.fields_by_name["numericData"].containing_oneof = _TIMESERIESDATA.oneofs_by_name["data"] -_NAMEDTIMESERIESDATA.fields_by_name["stringData"].message_type = _STRINGTIMESERIESDATA -_NAMEDTIMESERIESDATA.fields_by_name["numericData"].message_type = _NUMERICTIMESERIESDATA -_NAMEDTIMESERIESDATA.oneofs_by_name["data"].fields.append(_NAMEDTIMESERIESDATA.fields_by_name["stringData"]) -_NAMEDTIMESERIESDATA.fields_by_name["stringData"].containing_oneof = _NAMEDTIMESERIESDATA.oneofs_by_name["data"] -_NAMEDTIMESERIESDATA.oneofs_by_name["data"].fields.append(_NAMEDTIMESERIESDATA.fields_by_name["numericData"]) -_NAMEDTIMESERIESDATA.fields_by_name["numericData"].containing_oneof = _NAMEDTIMESERIESDATA.oneofs_by_name["data"] -_MULTINAMEDTIMESERIESDATA.fields_by_name["namedTimeseriesData"].message_type = _NAMEDTIMESERIESDATA -DESCRIPTOR.message_types_by_name["StringDatapoint"] = _STRINGDATAPOINT -DESCRIPTOR.message_types_by_name["NumericDatapoint"] = _NUMERICDATAPOINT -DESCRIPTOR.message_types_by_name["StringTimeseriesData"] = _STRINGTIMESERIESDATA -DESCRIPTOR.message_types_by_name["NumericTimeseriesData"] = _NUMERICTIMESERIESDATA -DESCRIPTOR.message_types_by_name["TimeseriesData"] = _TIMESERIESDATA -DESCRIPTOR.message_types_by_name["NamedTimeseriesData"] = _NAMEDTIMESERIESDATA -DESCRIPTOR.message_types_by_name["MultiNamedTimeseriesData"] = _MULTINAMEDTIMESERIESDATA -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -StringDatapoint = _reflection.GeneratedProtocolMessageType( - "StringDatapoint", - (_message.Message,), - dict( - DESCRIPTOR=_STRINGDATAPOINT, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.StringDatapoint) - ), -) -_sym_db.RegisterMessage(StringDatapoint) - -NumericDatapoint = _reflection.GeneratedProtocolMessageType( - "NumericDatapoint", - (_message.Message,), - dict( - DESCRIPTOR=_NUMERICDATAPOINT, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.NumericDatapoint) - ), -) -_sym_db.RegisterMessage(NumericDatapoint) - -StringTimeseriesData = _reflection.GeneratedProtocolMessageType( - "StringTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_STRINGTIMESERIESDATA, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.StringTimeseriesData) - ), -) -_sym_db.RegisterMessage(StringTimeseriesData) - -NumericTimeseriesData = _reflection.GeneratedProtocolMessageType( - "NumericTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_NUMERICTIMESERIESDATA, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.NumericTimeseriesData) - ), -) -_sym_db.RegisterMessage(NumericTimeseriesData) - -TimeseriesData = _reflection.GeneratedProtocolMessageType( - "TimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_TIMESERIESDATA, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.TimeseriesData) - ), -) -_sym_db.RegisterMessage(TimeseriesData) - -NamedTimeseriesData = _reflection.GeneratedProtocolMessageType( - "NamedTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_NAMEDTIMESERIESDATA, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.NamedTimeseriesData) - ), -) -_sym_db.RegisterMessage(NamedTimeseriesData) - -MultiNamedTimeseriesData = _reflection.GeneratedProtocolMessageType( - "MultiNamedTimeseriesData", - (_message.Message,), - dict( - DESCRIPTOR=_MULTINAMEDTIMESERIESDATA, - __module__="api_timeseries_data_v2_pb2" - # @@protoc_insertion_point(class_scope:api.v2.MultiNamedTimeseriesData) - ), -) -_sym_db.RegisterMessage(MultiNamedTimeseriesData) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions( - descriptor_pb2.FileOptions(), _b("\n\027com.cognite.data.api.v2P\001\252\002\023Cognite.Data.Api.V2") -) -# @@protoc_insertion_point(module_scope) diff --git a/cognite/client/_cognite_client.py b/cognite/client/_cognite_client.py new file mode 100644 index 0000000000..4ba7419f45 --- /dev/null +++ b/cognite/client/_cognite_client.py @@ -0,0 +1,251 @@ +import logging +import os +import sys +import warnings +from typing import Any, Dict + +from cognite.client._api.assets import AssetsAPI +from cognite.client._api.datapoints import DatapointsAPI +from cognite.client._api.events import EventsAPI +from cognite.client._api.files import FilesAPI +from cognite.client._api.iam import IAMAPI +from cognite.client._api.login import LoginAPI +from cognite.client._api.raw import RawAPI +from cognite.client._api.three_d import ThreeDAPI +from cognite.client._api.time_series import TimeSeriesAPI +from cognite.client._api_client import APIClient +from cognite.client.exceptions import CogniteAPIKeyError +from cognite.client.utils._utils import DebugLogFormatter, get_current_sdk_version +from cognite.client.utils._version_checker import get_newest_version_in_major_release + +DEFAULT_BASE_URL = "https://api.cognitedata.com" +DEFAULT_MAX_WORKERS = 10 +DEFAULT_TIMEOUT = 30 + + +class CogniteClient: + """Main entrypoint into Cognite Python SDK. + + All services are made available through this object. See examples below. + + Args: + api_key (str): API key + project (str): Project. Defaults to project of given API key. + client_name (str): A user-defined name for the client. Used to identify number of unique applications/scripts + running on top of CDF. + base_url (str): Base url to send requests to. Defaults to "https://api.cognitedata.com" + max_workers (int): Max number of workers to spawn when parallelizing data fetching. Defaults to 10. + headers (Dict): Additional headers to add to all requests. + timeout (int): Timeout on requests sent to the api. Defaults to 30 seconds. + debug (bool): Configures logger to log extra request details to stderr. + """ + + def __init__( + self, + api_key: str = None, + project: str = None, + client_name: str = None, + base_url: str = None, + max_workers: int = None, + headers: Dict[str, str] = None, + timeout: int = None, + debug: bool = None, + ): + thread_local_api_key, thread_local_project = self._get_thread_local_credentials() + environment_api_key = os.getenv("COGNITE_API_KEY") + environment_base_url = os.getenv("COGNITE_BASE_URL") + environment_max_workers = os.getenv("COGNITE_MAX_WORKERS") + environment_timeout = os.getenv("COGNITE_TIMEOUT") + environment_client_name = os.getenv("COGNITE_CLIENT_NAME") + + self.__api_key = api_key or thread_local_api_key or environment_api_key + if self.__api_key is None: + raise ValueError("No API Key has been specified") + + self._base_url = base_url or environment_base_url or DEFAULT_BASE_URL + + self._max_workers = int(max_workers or environment_max_workers or DEFAULT_MAX_WORKERS) + + self._headers = headers or {} + + self._client_name = client_name if client_name is not None else environment_client_name + if self._client_name is None: + raise ValueError( + "No client name has been specified. Pass it to the CogniteClient or set the environment variable 'COGNITE_CLIENT_NAME'." + ) + self._headers["x-cdp-app"] = client_name + + self._timeout = int(timeout or environment_timeout or DEFAULT_TIMEOUT) + + if debug: + self._configure_logger_for_debug_mode() + + __api_version = "v1" + + self.project = project or thread_local_project + self.login = LoginAPI( + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + + if self.project is None: + login_status = self.login.status() + if login_status.logged_in: + self.project = login_status.project + warnings.warn( + "Authenticated towards inferred project '{}'. Pass project to the CogniteClient constructor" + " to suppress this warning.".format(self.project), + stacklevel=2, + ) + else: + raise CogniteAPIKeyError + self._check_client_has_newest_major_version() + + self.assets = AssetsAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.datapoints = DatapointsAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.events = EventsAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.files = FilesAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.iam = IAMAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.time_series = TimeSeriesAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.raw = RawAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self.three_d = ThreeDAPI( + version=__api_version, + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + self._api_client = APIClient( + project=self.project, + api_key=self.__api_key, + base_url=self._base_url, + max_workers=self._max_workers, + headers=self._headers, + timeout=self._timeout, + cognite_client=self, + ) + + def get(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + """Perform a GET request to an arbitrary path in the API.""" + return self._api_client._get(url, params=params, headers=headers) + + def post(self, url: str, json: Dict[str, Any], params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + """Perform a POST request to an arbitrary path in the API.""" + return self._api_client._post(url, json=json, params=params, headers=headers) + + def put(self, url: str, json: Dict[str, Any] = None, headers: Dict[str, Any] = None): + """Perform a PUT request to an arbitrary path in the API.""" + return self._api_client._put(url, json=json, headers=headers) + + def delete(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): + """Perform a DELETE request to an arbitrary path in the API.""" + return self._api_client._delete(url, params=params, headers=headers) + + @property + def version(self) -> str: + """Returns the current SDK version. + + Returns: + str: The current SDK version + """ + return get_current_sdk_version() + + @staticmethod + def _get_thread_local_credentials(): + if "cognite._thread_local" in sys.modules: + from cognite._thread_local import credentials + + thread_local_api_key = getattr(credentials, "api_key", None) + thread_local_project = getattr(credentials, "project", None) + return thread_local_api_key, thread_local_project + return None, None + + def _configure_logger_for_debug_mode(self): + logger = logging.getLogger("cognite-sdk") + logger.setLevel("DEBUG") + log_handler = logging.StreamHandler() + formatter = DebugLogFormatter() + log_handler.setFormatter(formatter) + logger.handlers = [] + logger.propagate = False + logger.addHandler(log_handler) + + def _check_client_has_newest_major_version(self): + newest_version = get_newest_version_in_major_release("cognite-sdk", self.version) + if newest_version != self.version: + warnings.warn( + "You are using version {} of the SDK, however version {} is available. " + "Upgrade to suppress this warning.".format(self.version, newest_version), + stacklevel=3, + ) diff --git a/cognite/client/_utils.py b/cognite/client/_utils.py deleted file mode 100644 index f2512913fb..0000000000 --- a/cognite/client/_utils.py +++ /dev/null @@ -1,168 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utilites for Cognite API SDK - -This module provides helper methods and different utilities for the Cognite API Python SDK. - -This module is protected and should not used by end-users. -""" -import datetime -import platform -import re -import time -from datetime import datetime, timezone -from typing import Callable, List - -import cognite.client - - -def datetime_to_ms(dt): - return int(dt.replace(tzinfo=timezone.utc).timestamp() * 1000) - - -def granularity_to_ms(time_string): - """Returns millisecond representation of granularity time string""" - magnitude = int("".join([c for c in time_string if c.isdigit()])) - unit = "".join([c for c in time_string if c.isalpha()]) - unit_in_ms = { - "s": 1000, - "second": 1000, - "m": 60000, - "minute": 60000, - "h": 3600000, - "hour": 3600000, - "d": 86400000, - "day": 86400000, - } - return magnitude * unit_in_ms[unit] - - -def _time_ago_to_ms(time_ago_string): - """Returns millisecond representation of time-ago string""" - if time_ago_string == "now": - return 0 - pattern = r"(\d+)([a-z])-ago" - res = re.match(pattern, str(time_ago_string)) - if res: - magnitude = int(res.group(1)) - unit = res.group(2) - unit_in_ms = {"s": 1000, "m": 60000, "h": 3600000, "d": 86400000, "w": 604800000} - return magnitude * unit_in_ms[unit] - return None - - -def interval_to_ms(start, end): - """Returns the ms representation of start-end-interval whether it is time-ago, datetime or None.""" - time_now = int(round(time.time() * 1000)) - if isinstance(start, datetime): - start = datetime_to_ms(start) - elif isinstance(start, str): - start = time_now - _time_ago_to_ms(start) - elif start is None: - start = time_now - _time_ago_to_ms("2w-ago") - - if isinstance(end, datetime): - end = datetime_to_ms(end) - elif isinstance(end, str): - end = time_now - _time_ago_to_ms(end) - elif end is None: - end = time_now - - return start, end - - -class Bin: - """ - Attributes: - entries (List): List of entries. - get_count (Callable): Callable function to get count. - """ - - def __init__(self, get_count): - """ - Args: - get_count: A function that will take an element and get the count of something in it. - """ - self.entries = [] - self.get_count = get_count - - def add_item(self, item): - self.entries.append(item) - - def sum(self): - total = 0 - for elem in self.entries: - total += self.get_count(elem) - return total - - def show(self): - return self.entries - - -def first_fit(list_items: List, max_size, get_count: Callable) -> List[List]: - """Returns list of bins with input items inside.""" - - # Sort the input list in decreasing order - list_items = sorted(list_items, key=get_count, reverse=True) - - list_bins = [Bin(get_count=get_count)] - - for item in list_items: - # Go through bins and try to allocate - alloc_flag = False - - for bin in list_bins: - if bin.sum() + get_count(item) <= max_size: - bin.add_item(item) - alloc_flag = True - break - - # If item not allocated in bins in list, create new bin - # and allocate it to it. - if not alloc_flag: - new_bin = Bin(get_count=get_count) - new_bin.add_item(item) - list_bins.append(new_bin) - - # Turn bins into list of items and return - list_items = [] - for bin in list_bins: - list_items.append(bin.show()) - - return list_items - - -def get_user_agent(): - sdk_version = "CognitePythonSDK/{}".format(cognite.client.__version__) - - python_version = "{}/{} ({};{})".format( - platform.python_implementation(), platform.python_version(), platform.python_build(), platform.python_compiler() - ) - - os_version_info = [platform.release(), platform.machine(), platform.architecture()[0]] - os_version_info = [s for s in os_version_info if s] # Ignore empty strings - os_version_info = "-".join(os_version_info) - operating_system = "{}/{}".format(platform.system(), os_version_info) - - return "{} {} {}".format(sdk_version, python_version, operating_system) - - -def _round_to_nearest(x, base): - return int(base * round(float(x) / base)) - - -def get_datapoints_windows(start: int, end: int, granularity: str, num_of_workers): - diff = end - start - granularity_ms = 1 - if granularity: - granularity_ms = granularity_to_ms(granularity) - - # Ensure that number of steps is not greater than the number data points that will be returned - steps = min(num_of_workers, max(1, int(diff / granularity_ms))) - # Make step size a multiple of the granularity requested in order to ensure evenly spaced results - step_size = _round_to_nearest(int(diff / steps), base=granularity_ms) - # Create list of where each of the parallelized intervals will begin - step_starts = [start + (i * step_size) for i in range(steps)] - windows = [{"start": start, "end": start + step_size} for start in step_starts] - if windows[-1]["end"] < end: - windows[-1]["end"] = end - return windows diff --git a/cognite/client/cognite_client.py b/cognite/client/cognite_client.py deleted file mode 100644 index a6d46e1ee9..0000000000 --- a/cognite/client/cognite_client.py +++ /dev/null @@ -1,214 +0,0 @@ -import os -import sys -from typing import Any, Dict - -import requests - -from cognite.client._api_client import APIClient -from cognite.client._utils import get_user_agent -from cognite.client.experimental import ExperimentalClient -from cognite.client.stable.assets import AssetsClient -from cognite.client.stable.datapoints import DatapointsClient -from cognite.client.stable.events import EventsClient -from cognite.client.stable.files import FilesClient -from cognite.client.stable.login import LoginClient -from cognite.client.stable.raw import RawClient -from cognite.client.stable.tagmatching import TagMatchingClient -from cognite.client.stable.time_series import TimeSeriesClient -from cognite.logger import configure_logger - -DEFAULT_BASE_URL = "https://api.cognitedata.com" -DEFAULT_NUM_OF_WORKERS = 10 -DEFAULT_TIMEOUT = 30 - - -class CogniteClient: - """Main entrypoint into Cognite Python SDK. - - All services are made available through this object. See examples below. - - Args: - api_key (str): API key - project (str): Project. Defaults to project of given API key. - base_url (str): Base url to send requests to. Defaults to "https://api.cognitedata.com" - num_of_workers (int): Number of workers to spawn when parallelizing data fetching. Defaults to 10. - cookies (Dict): Cookies to append to all requests. Defaults to {} - headers (Dict): Additional headers to add to all requests. Defaults are: - {"api-key": self.api_key, "content-type": "application/json", "accept": "application/json"} - timeout (int): Timeout on requests sent to the api. Defaults to 60 seconds. - debug (bool): Configures logger to log extra request details to stdout. - - - Examples: - The CogniteClient is instantiated and used like this. This example assumes that the environment variable - COGNITE_API_KEY has been set:: - - from cognite.client import CogniteClient - client = CogniteClient() - res = client.time_series.get_time_series() - print(res.to_pandas()) - - Certain experimental features are made available through this client as follows:: - - from cognite.client import CogniteClient - client = CogniteClient() - res = client.experimental.model_hosting.models.list_models() - print(res) - - Default configurations may be set using the following environment variables:: - - export COGNITE_API_KEY = - export COGNITE_BASE_URL = http://: - export COGNITE_NUM_RETRIES = - export COGNITE_NUM_WORKERS = - export COGNITE_TIMEOUT = - export COGNITE_DISABLE_GZIP = "1" - """ - - def __init__( - self, - api_key: str = None, - project: str = None, - base_url: str = None, - num_of_workers: int = None, - headers: Dict[str, str] = None, - cookies: Dict[str, str] = None, - timeout: int = None, - debug: bool = None, - ): - thread_local_api_key, thread_local_project = self._get_thread_local_credentials() - - environment_api_key = os.getenv("COGNITE_API_KEY") - environment_base_url = os.getenv("COGNITE_BASE_URL") - environment_num_of_workers = os.getenv("COGNITE_NUM_WORKERS") - environment_timeout = os.getenv("COGNITE_TIMEOUT") - - self.__api_key = api_key or thread_local_api_key or environment_api_key - if self.__api_key is None: - raise ValueError("No Api Key has been specified") - - self._base_url = base_url or environment_base_url or DEFAULT_BASE_URL - - self._num_of_workers = int(num_of_workers or environment_num_of_workers or DEFAULT_NUM_OF_WORKERS) - - self._configure_headers(headers) - - self._cookies = cookies or {} - - self._timeout = int(timeout or environment_timeout or DEFAULT_TIMEOUT) - - self._project = project or thread_local_project - self._login_client = self._client_factory(LoginClient) - if self._project is None: - self._project = self._login_client.status().project - - self._api_client = self._client_factory(APIClient) - - if debug: - configure_logger("cognite-sdk", log_level="INFO", log_json=True) - - self._assets_client = self._client_factory(AssetsClient) - self._datapoints_client = self._client_factory(DatapointsClient) - self._events_client = self._client_factory(EventsClient) - self._files_client = self._client_factory(FilesClient) - self._raw_client = self._client_factory(RawClient) - self._tagmatching_client = self._client_factory(TagMatchingClient) - self._time_series_client = self._client_factory(TimeSeriesClient) - self._experimental_client = ExperimentalClient(self._client_factory) - - @property - def assets(self) -> AssetsClient: - return self._assets_client - - @property - def datapoints(self) -> DatapointsClient: - return self._datapoints_client - - @property - def events(self) -> EventsClient: - return self._events_client - - @property - def files(self) -> FilesClient: - return self._files_client - - @property - def login(self) -> LoginClient: - return self._login_client - - @property - def raw(self) -> RawClient: - return self._raw_client - - @property - def tag_matching(self) -> TagMatchingClient: - return self._tagmatching_client - - @property - def time_series(self) -> TimeSeriesClient: - return self._time_series_client - - @property - def experimental(self) -> ExperimentalClient: - return self._experimental_client - - def get(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None, autopaging: bool = False): - """Perform a GET request to a path in the API. - - Comes in handy if the endpoint you want to reach is not currently supported by the SDK. - """ - return self._api_client._get(url, params=params, headers=headers, autopaging=autopaging) - - def post(self, url: str, body: Dict[str, Any], params: Dict[str, Any] = None, headers: Dict[str, Any] = None): - """Perform a POST request to a path in the API. - - Comes in handy if the endpoint you want to reach is not currently supported by the SDK. - """ - return self._api_client._post(url, body=body, params=params, headers=headers) - - def put(self, url: str, body: Dict[str, Any] = None, headers: Dict[str, Any] = None): - """Perform a PUT request to a path in the API. - - Comes in handy if the endpoint you want to reach is not currently supported by the SDK. - """ - return self._api_client._put(url, body=body, headers=headers) - - def delete(self, url: str, params: Dict[str, Any] = None, headers: Dict[str, Any] = None): - """Perform a DELETE request to a path in the API. - - Comes in handy if the endpoint you want to reach is not currently supported by the SDK. - """ - return self._api_client._delete(url, params=params, headers=headers) - - def _client_factory(self, client): - return client( - project=self._project, - base_url=self._base_url, - num_of_workers=self._num_of_workers, - cookies=self._cookies, - headers=self._headers, - timeout=self._timeout, - ) - - def _configure_headers(self, user_defined_headers): - self._headers = requests.utils.default_headers() - self._headers.update( - {"api-key": self.__api_key, "content-type": "application/json", "accept": "application/json"} - ) - - if "User-Agent" in self._headers: - self._headers["User-Agent"] += " " + get_user_agent() - else: - self._headers["User-Agent"] = get_user_agent() - - if user_defined_headers: - self._headers.update(user_defined_headers) - - def _get_thread_local_credentials(self): - if "cognite._thread_local" in sys.modules: - from cognite._thread_local import credentials - - thread_local_api_key = getattr(credentials, "api_key", None) - thread_local_project = getattr(credentials, "project", None) - return thread_local_api_key, thread_local_project - return None, None diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py new file mode 100644 index 0000000000..e3c9bd2cfd --- /dev/null +++ b/cognite/client/data_classes/__init__.py @@ -0,0 +1,35 @@ +from typing import * + +from cognite.client.data_classes.assets import Asset, AssetFilter, AssetList, AssetUpdate +from cognite.client.data_classes.datapoints import Datapoint, Datapoints, DatapointsList, DatapointsQuery +from cognite.client.data_classes.events import Event, EventFilter, EventList, EventUpdate +from cognite.client.data_classes.files import FileMetadata, FileMetadataFilter, FileMetadataList, FileMetadataUpdate +from cognite.client.data_classes.iam import ( + APIKey, + APIKeyList, + Group, + GroupList, + SecurityCategory, + SecurityCategoryList, + ServiceAccount, + ServiceAccountList, +) +from cognite.client.data_classes.raw import Database, DatabaseList, Row, RowList, Table, TableList +from cognite.client.data_classes.three_d import ( + ThreeDAssetMapping, + ThreeDAssetMappingList, + ThreeDModel, + ThreeDModelList, + ThreeDModelRevision, + ThreeDModelRevisionList, + ThreeDModelRevisionUpdate, + ThreeDModelUpdate, + ThreeDNode, + ThreeDNodeList, + ThreeDRevealNode, + ThreeDRevealNodeList, + ThreeDRevealRevision, + ThreeDRevealSector, + ThreeDRevealSectorList, +) +from cognite.client.data_classes.time_series import TimeSeries, TimeSeriesFilter, TimeSeriesList, TimeSeriesUpdate diff --git a/cognite/client/data_classes/_base.py b/cognite/client/data_classes/_base.py new file mode 100644 index 0000000000..62f2434883 --- /dev/null +++ b/cognite/client/data_classes/_base.py @@ -0,0 +1,361 @@ +import functools +import json +from collections import UserList +from typing import * + +from cognite.client.exceptions import CogniteMissingClientError +from cognite.client.utils import _utils as utils +from cognite.client.utils._utils import to_camel_case, to_snake_case + +EXCLUDE_VALUE = [None] + + +class CogniteResponse: + def __str__(self): + item = utils.convert_time_attributes_to_datetime(self.dump()) + return json.dumps(item, indent=4) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return type(other) == type(self) and other.dump() == self.dump() + + def __getattribute__(self, item): + attr = super().__getattribute__(item) + if item == "_cognite_client": + if attr is None: + raise CogniteMissingClientError + return attr + + def dump(self, camel_case: bool = False) -> Dict[str, Any]: + """Dump the instance into a json serializable python data type. + + Args: + camel_case (bool): Use camelCase for attribute names. Defaults to False. + + Returns: + Dict[str, Any]: A dictionary representation of the instance. + """ + dumped = { + key: value for key, value in self.__dict__.items() if value not in EXCLUDE_VALUE and not key.startswith("_") + } + if camel_case: + dumped = {to_camel_case(key): value for key, value in dumped.items()} + return dumped + + @classmethod + def _load(cls, api_response): + raise NotImplementedError + + def to_pandas(self): + raise NotImplementedError + + +class CogniteResource: + def __new__(cls, *args, **kwargs): + obj = super().__new__(cls) + obj._cognite_client = None + if "cognite_client" in kwargs: + obj._cognite_client = kwargs["cognite_client"] + return obj + + def __eq__(self, other): + return type(self) == type(other) and self.dump() == other.dump() + + def __str__(self): + item = utils.convert_time_attributes_to_datetime(self.dump()) + return json.dumps(item, default=lambda x: x.__dict__, indent=4) + + def __repr__(self): + return self.__str__() + + def __getattribute__(self, item): + attr = super().__getattribute__(item) + if item == "_cognite_client": + if attr is None: + raise CogniteMissingClientError + return attr + + def dump(self, camel_case: bool = False) -> Dict[str, Any]: + """Dump the instance into a json serializable Python data type. + + Args: + camel_case (bool): Use camelCase for attribute names. Defaults to False. + + Returns: + Dict[str, Any]: A dictionary representation of the instance. + """ + if camel_case: + return { + to_camel_case(key): value + for key, value in self.__dict__.items() + if value not in EXCLUDE_VALUE and not key.startswith("_") + } + return { + key: value for key, value in self.__dict__.items() if value not in EXCLUDE_VALUE and not key.startswith("_") + } + + @classmethod + def _load(cls, resource: Union[Dict, str], cognite_client=None): + if isinstance(resource, str): + return cls._load(json.loads(resource), cognite_client=cognite_client) + elif isinstance(resource, Dict): + instance = cls(cognite_client=cognite_client) + for key, value in resource.items(): + snake_case_key = to_snake_case(key) + if not hasattr(instance, snake_case_key): + raise AttributeError("Attribute '{}' does not exist on '{}'".format(snake_case_key, cls.__name__)) + setattr(instance, snake_case_key, value) + return instance + raise TypeError("Resource must be json str or Dict, not {}".format(type(resource))) + + def to_pandas(self, expand: List[str] = None, ignore: List[str] = None): + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The dataframe. + """ + expand = ["metadata"] if expand is None else expand + ignore = [] if ignore is None else ignore + pd = utils.local_import("pandas") + dumped = self.dump(camel_case=True) + + for element in ignore: + del dumped[element] + for key in expand: + if key in dumped and isinstance(dumped[key], dict): + dumped.update(dumped.pop(key)) + else: + raise AssertionError("Could not expand attribute '{}'".format(key)) + + df = pd.DataFrame(columns=["value"]) + for name, value in dumped.items(): + df.loc[name] = [value] + return df + + +class CogniteResourceList(UserList): + _RESOURCE = None + _UPDATE = None + _ASSERT_CLASSES = True + + def __init__(self, resources: List[Any], cognite_client=None): + if self._ASSERT_CLASSES: + assert self._RESOURCE is not None, "{} does not have _RESOURCE set".format(self.__class__.__name__) + assert self._UPDATE is not None, "{} does not have _UPDATE set".format(self.__class__.__name__) + for resource in resources: + if not isinstance(resource, self._RESOURCE): + raise TypeError( + "All resources for class '{}' must be of type '{}', not '{}'.".format( + self.__class__.__name__, self._RESOURCE.__name__, type(resource) + ) + ) + self._cognite_client = cognite_client + super().__init__(resources) + if self.data: + if hasattr(self.data[0], "external_id"): + self._external_id_to_item = { + item.external_id: item for item in self.data if item.external_id is not None + } + if hasattr(self.data[0], "id"): + self._id_to_item = {item.id: item for item in self.data if item.id is not None} + + def __getattribute__(self, item): + attr = super().__getattribute__(item) + if item == "_cognite_client" and attr is None: + raise CogniteMissingClientError + return attr + + def __getitem__(self, item): + value = super().__getitem__(item) + if isinstance(item, slice): + c = None + if super().__getattribute__("_cognite_client") is not None: + c = self._cognite_client + return self.__class__(value, cognite_client=c) + return value + + def __str__(self): + item = utils.convert_time_attributes_to_datetime(self.dump()) + return json.dumps(item, default=lambda x: x.__dict__, indent=4) + + def __repr__(self): + return self.__str__() + + def dump(self, camel_case: bool = False) -> List[Dict[str, Any]]: + """Dump the instance into a json serializable Python data type. + + Args: + camel_case (bool): Use camelCase for attribute names. Defaults to False. + + Returns: + List[Dict[str, Any]]: A list of dicts representing the instance. + """ + return [resource.dump(camel_case) for resource in self.data] + + def get(self, id: int = None, external_id: str = None) -> Optional[CogniteResource]: + """Get an item from this list by id or exernal_id. + + Args: + id (int): The id of the item to get. + external_id (str): The external_id of the item to get. + + Returns: + Optional[CogniteResource]: The requested item + """ + utils.assert_exactly_one_of_id_or_external_id(id, external_id) + if id: + return self._id_to_item.get(id) + return self._external_id_to_item.get(external_id) + + def to_pandas(self) -> "pandas.DataFrame": + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The dataframe. + """ + pd = utils.local_import("pandas") + df = pd.DataFrame(self.dump(camel_case=True)) + nullable_int_fields = ["endTime", "assetId"] + for field in nullable_int_fields: + if field in df: + df[field] = df[field].astype(pd.Int64Dtype()) + return df + + @classmethod + def _load(cls, resource_list: Union[List, str], cognite_client=None): + if isinstance(resource_list, str): + return cls._load(json.loads(resource_list), cognite_client=cognite_client) + elif isinstance(resource_list, List): + resources = [cls._RESOURCE._load(resource, cognite_client=cognite_client) for resource in resource_list] + return cls(resources, cognite_client=cognite_client) + + +class CogniteUpdate: + def __init__(self, id: int = None, external_id: str = None): + self._id = id + self._external_id = external_id + self._update_object = {} + + def __eq__(self, other): + return type(self) == type(other) and self.dump() == other.dump() + + def __str__(self): + return json.dumps(self.dump(), indent=4) + + def __repr__(self): + return self.__str__() + + def _set(self, name, value): + self._update_object[name] = {"set": value} + + def _set_null(self, name): + self._update_object[name] = {"setNull": True} + + def _add(self, name, value): + self._update_object[name] = {"add": value} + + def _remove(self, name, value): + self._update_object[name] = {"remove": value} + + def dump(self): + """Dump the instance into a json serializable Python data type. + + Returns: + Dict[str, Any]: A dictionary representation of the instance. + """ + dumped = {"update": self._update_object} + if self._id is not None: + dumped["id"] = self._id + elif self._external_id is not None: + dumped["externalId"] = self._external_id + return dumped + + @classmethod + def _get_update_properties(cls): + return [key for key in cls.__dict__.keys() if not key.startswith("_")] + + +class CognitePrimitiveUpdate: + def __init__(self, update_object, name: str): + self._update_object = update_object + self._name = name + + def _set(self, value: Union[None, str, int, bool]): + if value is None: + self._update_object._set_null(self._name) + else: + self._update_object._set(self._name, value) + return self._update_object + + +class CogniteObjectUpdate: + def __init__(self, update_object, name: str): + self._update_object = update_object + self._name = name + + def _set(self, value: Dict): + self._update_object._set(self._name, value) + return self._update_object + + def _add(self, value: Dict): + self._update_object._add(self._name, value) + return self._update_object + + def _remove(self, value: List): + self._update_object._remove(self._name, value) + return self._update_object + + +class CogniteListUpdate: + def __init__(self, update_object, name: str): + self._update_object = update_object + self._name = name + + def _set(self, value: List): + self._update_object._set(self._name, value) + return self._update_object + + def _add(self, value: List): + self._update_object._add(self._name, value) + return self._update_object + + def _remove(self, value: List): + self._update_object._remove(self._name, value) + return self._update_object + + +class CogniteFilter: + def __eq__(self, other): + return type(self) == type(other) and self.dump() == other.dump() + + def __str__(self): + item = utils.convert_time_attributes_to_datetime(self.dump()) + return json.dumps(item, default=lambda x: x.__dict__, indent=4) + + def __repr__(self): + return self.__str__() + + def __getattribute__(self, item): + attr = super().__getattribute__(item) + if item == "_cognite_client": + if attr is None: + raise CogniteMissingClientError + return attr + + def dump(self, camel_case: bool = False): + """Dump the instance into a json serializable Python data type. + + Returns: + Dict[str, Any]: A dictionary representation of the instance. + """ + if camel_case: + return { + to_camel_case(key): value + for key, value in self.__dict__.items() + if value not in EXCLUDE_VALUE and not key.startswith("_") + } + return { + key: value for key, value in self.__dict__.items() if value not in EXCLUDE_VALUE and not key.startswith("_") + } diff --git a/cognite/client/data_classes/assets.py b/cognite/client/data_classes/assets.py new file mode 100644 index 0000000000..c7e6ca9a42 --- /dev/null +++ b/cognite/client/data_classes/assets.py @@ -0,0 +1,301 @@ +from cognite.client.data_classes._base import * + + +# GenClass: Asset, DataExternalAssetItem +class Asset(CogniteResource): + """Representation of a physical asset, e.g plant or piece of equipment + + Args: + external_id (str): External Id provided by client. Should be unique within the project. + name (str): Name of asset. Often referred to as tag. + parent_id (int): Javascript friendly internal ID given to the object. + description (str): Description of asset. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + id (int): Javascript friendly internal ID given to the object. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + path (List[int]): IDs of assets on the path to the asset. + depth (int): Asset path depth (number of levels below root node). + parent_external_id (str): External Id provided by client. Should be unique within the project. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + external_id: str = None, + name: str = None, + parent_id: int = None, + description: str = None, + metadata: Dict[str, Any] = None, + source: str = None, + id: int = None, + created_time: int = None, + last_updated_time: int = None, + path: List[int] = None, + depth: int = None, + parent_external_id: str = None, + cognite_client=None, + ): + self.external_id = external_id + self.name = name + self.parent_id = parent_id + self.description = description + self.metadata = metadata + self.source = source + self.id = id + self.created_time = created_time + self.last_updated_time = last_updated_time + self.path = path + self.depth = depth + self.parent_external_id = parent_external_id + self._cognite_client = cognite_client + + # GenStop + + def __hash__(self): + return hash(self.external_id) + + def parent(self) -> "Asset": + """Returns this assets parent. + + Returns: + Asset: The parent asset. + """ + if self.parent_id is None: + raise ValueError("parent_id is None") + return self._cognite_client.assets.retrieve(id=self.parent_id) + + def children(self) -> "AssetList": + """Returns the children of this asset. + + Returns: + AssetList: The requested assets + """ + return self._cognite_client.assets.list(parent_ids=[self.id], limit=None) + + def subtree(self, depth: int = None) -> "AssetList": + """Returns the subtree of this asset up to a specified depth. + + Args: + depth (int, optional): Retrieve assets up to this depth below the asset. + + Returns: + AssetList: The requested assets sorted topologically. + """ + return self._cognite_client.assets.retrieve_subtree(id=self.id, depth=depth) + + def time_series(self, **kwargs) -> "TimeSeriesList": + """Retrieve all time series related to this asset. + + Returns: + TimeSeriesList: All time series related to this asset. + """ + return self._cognite_client.time_series.list(asset_ids=[self.id], **kwargs) + + def events(self, **kwargs) -> "EventList": + """Retrieve all events related to this asset. + + Returns: + EventList: All events related to this asset. + """ + + return self._cognite_client.events.list(asset_ids=[self.id], **kwargs) + + def files(self, **kwargs) -> "FileMetadataList": + """Retrieve all files metadata related to this asset. + + Returns: + FileMetadataList: Metadata about all files related to this asset. + """ + return self._cognite_client.files.list(asset_ids=[self.id], **kwargs) + + +# GenUpdateClass: AssetChange +class AssetUpdate(CogniteUpdate): + """Changes applied to asset + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project. + """ + + @property + def external_id(self): + return _PrimitiveAssetUpdate(self, "externalId") + + @property + def name(self): + return _PrimitiveAssetUpdate(self, "name") + + @property + def description(self): + return _PrimitiveAssetUpdate(self, "description") + + @property + def metadata(self): + return _ObjectAssetUpdate(self, "metadata") + + @property + def source(self): + return _PrimitiveAssetUpdate(self, "source") + + +class _PrimitiveAssetUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> AssetUpdate: + return self._set(value) + + +class _ObjectAssetUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> AssetUpdate: + return self._set(value) + + def add(self, value: Dict) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + +class _ListAssetUpdate(CogniteListUpdate): + def set(self, value: List) -> AssetUpdate: + return self._set(value) + + def add(self, value: List) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + # GenStop + + +class AssetList(CogniteResourceList): + _RESOURCE = Asset + _UPDATE = AssetUpdate + + def _indented_asset_str(self, asset: Asset): + single_indent = " " * 8 + marked_indent = "|______ " + indent = len(asset.path) - 1 + + s = single_indent * (indent - 1) + if indent > 0: + s += marked_indent + s += str(asset.id) + "\n" + dumped = utils.convert_time_attributes_to_datetime(asset.dump()) + for key, value in sorted(dumped.items()): + if isinstance(value, dict): + s += single_indent * indent + "{}:\n".format(key) + for mkey, mvalue in sorted(value.items()): + s += single_indent * indent + " - {}: {}\n".format(mkey, mvalue) + elif key != "id": + s += single_indent * indent + key + ": " + str(value) + "\n" + + return s + + def __str__(self): + try: + sorted_assets = sorted(self.data, key=lambda x: x.path) + except: + return super().__str__() + + if len(sorted_assets) == 0: + return super().__str__() + + ids = set([asset.id for asset in sorted_assets]) + + s = "\n" + root = sorted_assets[0].path[0] + for asset in sorted_assets: + this_root = asset.path[0] + if this_root != root: + s += "\n" + "*" * 80 + "\n\n" + root = this_root + elif len(asset.path) > 1 and asset.path[-2] not in ids: + s += "\n" + "-" * 80 + "\n\n" + s += self._indented_asset_str(asset) + return s + + def time_series(self) -> "TimeSeriesList": + """Retrieve all time series related to these assets. + + Returns: + TimeSeriesList: All time series related to the assets in this AssetList. + """ + from cognite.client.data_classes import TimeSeriesList + + return self._retrieve_related_resources(TimeSeriesList, self._cognite_client.time_series) + + def events(self) -> "EventList": + """Retrieve all events related to these assets. + + Returns: + EventList: All events related to the assets in this AssetList. + """ + from cognite.client.data_classes import EventList + + return self._retrieve_related_resources(EventList, self._cognite_client.events) + + def files(self) -> "FileMetadataList": + """Retrieve all files metadata related to these assets. + + Returns: + FileMetadataList: Metadata about all files related to the assets in this AssetList. + """ + from cognite.client.data_classes import FileMetadataList + + return self._retrieve_related_resources(FileMetadataList, self._cognite_client.files) + + def _retrieve_related_resources(self, resource_list_class, resource_api): + ids = [a.id for a in self.data] + tasks = [] + chunk_size = 100 + for i in range(0, len(ids), chunk_size): + tasks.append({"asset_ids": ids[i : i + chunk_size], "limit": -1}) + res_list = utils.execute_tasks_concurrently(resource_api.list, tasks, resource_api._max_workers).results + resources = resource_list_class([]) + for res in res_list: + resources.extend(res) + return resources + + +# GenClass: AssetFilter.filter +class AssetFilter(CogniteFilter): + """No description. + + Args: + name (str): Name of asset. Often referred to as tag. + parent_ids (List[int]): No description. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + root (bool): filtered assets are root assets or not + external_id_prefix (str): External Id provided by client. Should be unique within the project. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + parent_ids: List[int] = None, + metadata: Dict[str, Any] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + root: bool = None, + external_id_prefix: str = None, + cognite_client=None, + ): + self.name = name + self.parent_ids = parent_ids + self.metadata = metadata + self.source = source + self.created_time = created_time + self.last_updated_time = last_updated_time + self.root = root + self.external_id_prefix = external_id_prefix + self._cognite_client = cognite_client + + # GenStop diff --git a/cognite/client/data_classes/datapoints.py b/cognite/client/data_classes/datapoints.py new file mode 100644 index 0000000000..0adec303be --- /dev/null +++ b/cognite/client/data_classes/datapoints.py @@ -0,0 +1,342 @@ +from datetime import datetime + +from cognite.client.data_classes._base import * + + +class Datapoint(CogniteResource): + """An object representing a datapoint. + + Args: + timestamp (Union[int, float]): The data timestamp in milliseconds since the epoch (Jan 1, 1970). + value (Union[str, int, float]): The data value. Can be String or numeric depending on the metric + average (float): The integral average value in the aggregate period + max (float): The maximum value in the aggregate period + min (float): The minimum value in the aggregate period + count (int): The number of datapoints in the aggregate period + sum (float): The sum of the datapoints in the aggregate period + interpolation (float): The interpolated value of the series in the beginning of the aggregate + step_interpolation (float): The last value before or at the beginning of the aggregate. + continuous_variance (float): The variance of the interpolated underlying function. + discrete_variance (float): The variance of the datapoint values. + total_variation (float): The total variation of the interpolated underlying function. + """ + + def __init__( + self, + timestamp: Union[int, float] = None, + value: Union[str, int, float] = None, + average: float = None, + max: float = None, + min: float = None, + count: int = None, + sum: float = None, + interpolation: float = None, + step_interpolation: float = None, + continuous_variance: float = None, + discrete_variance: float = None, + total_variation: float = None, + ): + self.timestamp = timestamp + self.value = value + self.average = average + self.max = max + self.min = min + self.count = count + self.sum = sum + self.interpolation = interpolation + self.step_interpolation = step_interpolation + self.continuous_variance = continuous_variance + self.discrete_variance = discrete_variance + self.total_variation = total_variation + + def to_pandas(self) -> "pandas.DataFrame": + """Convert the datapoint into a pandas DataFrame. + + Returns: + pandas.DataFrame: The dataframe. + """ + pd = utils.local_import("pandas") + + dumped = self.dump(camel_case=True) + timestamp = dumped.pop("timestamp") + + for k, v in dumped.items(): + dumped[k] = [v] + df = pd.DataFrame(dumped, index=[utils.ms_to_datetime(timestamp)]) + + return df + + +class Datapoints: + """An object representing a list of datapoints. + + Args: + id (int): Id of the timeseries the datapoints belong to + external_id (str): External id of the timeseries the datapoints belong to (Only if id is not set) + timestamp (List[Union[int, float]]): The data timestamps in milliseconds since the epoch (Jan 1, 1970). + value (List[Union[int, str, float]]): The data values. Can be String or numeric depending on the metric + average (List[float]): The integral average values in the aggregate period + max (List[float]): The maximum values in the aggregate period + min (List[float]): The minimum values in the aggregate period + count (List[int]): The number of datapoints in the aggregate periods + sum (List[float]): The sum of the datapoints in the aggregate periods + interpolation (List[float]): The interpolated values of the series in the beginning of the aggregates + step_interpolation (List[float]): The last values before or at the beginning of the aggregates. + continuous_variance (List[float]): The variance of the interpolated underlying function. + discrete_variance (List[float]): The variance of the datapoint values. + total_variation (List[float]): The total variation of the interpolated underlying function. + """ + + def __init__( + self, + id: int = None, + external_id: str = None, + timestamp: List[Union[int, float]] = None, + value: List[Union[int, str, float]] = None, + average: List[float] = None, + max: List[float] = None, + min: List[float] = None, + count: List[int] = None, + sum: List[float] = None, + interpolation: List[float] = None, + step_interpolation: List[float] = None, + continuous_variance: List[float] = None, + discrete_variance: List[float] = None, + total_variation: List[float] = None, + ): + self.id = id + self.external_id = external_id + self.timestamp = timestamp or [] + self.value = value + self.average = average + self.max = max + self.min = min + self.count = count + self.sum = sum + self.interpolation = interpolation + self.step_interpolation = step_interpolation + self.continuous_variance = continuous_variance + self.discrete_variance = discrete_variance + self.total_variation = total_variation + + self.__datapoint_objects = None + + def __str__(self): + item = self.dump() + item["datapoints"] = utils.convert_time_attributes_to_datetime(item["datapoints"]) + return json.dumps(item, indent=4) + + def __repr__(self): + return self.__str__() + + def __len__(self) -> int: + return len(self.timestamp) + + def __eq__(self, other): + return ( + type(self) == type(other) + and self.id == other.id + and self.external_id == other.external_id + and list(self._get_non_empty_data_fields()) == list(other._get_non_empty_data_fields()) + ) + + def __getitem__(self, item) -> Union[Datapoint, "Datapoints"]: + if isinstance(item, slice): + return self._slice(item) + dp_args = {} + for attr, values in self._get_non_empty_data_fields(): + dp_args[attr] = values[item] + return Datapoint(**dp_args) + + def __iter__(self) -> Generator[Datapoint, None, None]: + yield from self.__get_datapoint_objects() + + def dump(self, camel_case: bool = False) -> Dict[str, Any]: + """Dump the datapoints into a json serializable Python data type. + + Args: + camel_case (bool): Use camelCase for attribute names. Defaults to False. + + Returns: + List[Dict[str, Any]]: A list of dicts representing the instance. + """ + dumped = { + "id": self.id, + "external_id": self.external_id, + "datapoints": [dp.dump(camel_case=camel_case) for dp in self.__get_datapoint_objects()], + } + if camel_case: + dumped = {utils.to_camel_case(key): value for key, value in dumped.items()} + return {key: value for key, value in dumped.items() if value is not None} + + def to_pandas(self, column_names="externalId") -> "pandas.DataFrame": + """Convert the datapoints into a pandas DataFrame. + + Args: + column_names (str): Which field to use as column header. Defaults to "externalId", can also be "id". + + Returns: + pandas.DataFrame: The dataframe. + """ + np, pd = utils.local_import("numpy", "pandas") + data_fields = {} + timestamps = [] + if column_names == "externalId": + identifier = self.external_id if self.external_id is not None else self.id + elif column_names == "id": + identifier = self.id + else: + raise ValueError("column_names must be 'externalId' or 'id'") + for attr, value in self._get_non_empty_data_fields(get_empty_lists=True): + if attr == "timestamp": + timestamps = value + else: + id_with_agg = str(identifier) + if attr != "value": + id_with_agg += "|{}".format(utils.to_camel_case(attr)) + data_fields[id_with_agg] = value + return pd.DataFrame(data_fields, index=pd.DatetimeIndex(data=np.array(timestamps, dtype="datetime64[ms]"))) + + def plot(self, *args, **kwargs) -> None: + """Plot the datapoints.""" + plt = utils.local_import("matplotlib.pyplot") + self.to_pandas().plot(*args, **kwargs) + plt.show() + + @classmethod + def _load(cls, dps_object, expected_fields: List[str] = None, cognite_client=None): + instance = cls() + instance.id = dps_object["id"] + instance.external_id = dps_object.get("externalId") + expected_fields = expected_fields or ["value"] + expected_fields.append("timestamp") + if len(dps_object["datapoints"]) == 0: + for key in expected_fields: + snake_key = utils.to_snake_case(key) + setattr(instance, snake_key, []) + else: + for dp in dps_object["datapoints"]: + for key in expected_fields: + snake_key = utils.to_snake_case(key) + current_attr = getattr(instance, snake_key) or [] + value = dp.get(key) + current_attr.append(value) + setattr(instance, snake_key, current_attr) + return instance + + def _insert(self, other_dps): + if self.id is None and self.external_id is None: + self.id = other_dps.id + self.external_id = other_dps.external_id + + if other_dps.timestamp: + other_first_ts = other_dps.timestamp[0] + index_to_split_on = None + for i, ts in enumerate(self.timestamp): + if ts > other_first_ts: + index_to_split_on = i + break + else: + index_to_split_on = 0 + + for attr, other_value in other_dps._get_non_empty_data_fields(get_empty_lists=True): + value = getattr(self, attr) + if not value: + setattr(self, attr, other_value) + else: + if index_to_split_on is not None: + new_value = value[:index_to_split_on] + other_value + value[index_to_split_on:] + else: + new_value = value + other_value + setattr(self, attr, new_value) + + def _get_non_empty_data_fields(self, get_empty_lists=False) -> List[Tuple[str, Any]]: + non_empty_data_fields = [] + for attr, value in self.__dict__.copy().items(): + if attr not in ["id", "external_id", "_Datapoints__datapoint_objects", "_cognite_client"]: + if value is not None or attr == "timestamp": + if len(value) > 0 or get_empty_lists or attr == "timestamp": + non_empty_data_fields.append((attr, value)) + return non_empty_data_fields + + def __get_datapoint_objects(self) -> List[Datapoint]: + if self.__datapoint_objects is None: + self.__datapoint_objects = [] + for i in range(len(self)): + dp_args = {} + for attr, value in self._get_non_empty_data_fields(): + dp_args[attr] = value[i] + self.__datapoint_objects.append(Datapoint(**dp_args)) + return self.__datapoint_objects + + def _slice(self, slice: slice): + truncated_datapoints = Datapoints(id=self.id, external_id=self.external_id) + for attr, value in self._get_non_empty_data_fields(): + setattr(truncated_datapoints, attr, value[slice]) + return truncated_datapoints + + +class DatapointsList(CogniteResourceList): + _RESOURCE = Datapoints + _ASSERT_CLASSES = False + + def __str__(self): + item = self.dump() + for i in item: + i["datapoints"] = utils.convert_time_attributes_to_datetime(i["datapoints"]) + return json.dumps(item, default=lambda x: x.__dict__, indent=4) + + def to_pandas(self, column_names="externalId") -> "pandas.DataFrame": + """Convert the datapoints list into a pandas DataFrame. + + Args: + column_names (str): Which field to use as column header. Defaults to "externalId", can also be "id". + Returns: + pandas.DataFrame: The datapoints list as a pandas DataFrame. + """ + pd = utils.local_import("pandas") + dfs = [df.to_pandas(column_names=column_names) for df in self.data] + if dfs: + return pd.concat(dfs, axis="columns") + return pd.DataFrame() + + def plot(self, *args, **kwargs) -> None: + """Plot the list of datapoints.""" + plt = utils.local_import("matplotlib.pyplot") + self.to_pandas().plot(*args, **kwargs) + plt.show() + + +class DatapointsQuery(CogniteResource): + """Parameters describing a query for datapoints. + + Args: + start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. Example: '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since epoch. + end (Union[str, int, datetime]): Get datapoints up to this time. The format is the same as for start. + id (int): Id of the timeseries to query + external_id (str): External id of the timeseries to query (Only if id is not set) + limit (int): Return up to this number of datapoints. + aggregates (List[str]): The aggregates to be returned. Use default if null. An empty string must be sent to get raw data if the default is a set of aggregates. + granularity (str): The granularity size and granularity of the aggregates. + include_outside_points (bool): Whether to include the last datapoint before the requested time period,and the first one after the requested period. This can be useful for interpolating data. Not available for aggregates. + """ + + def __init__( + self, + start: Union[str, int, datetime], + end: Union[str, int, datetime], + id: int = None, + external_id: str = None, + limit: int = None, + aggregates: List[str] = None, + granularity: str = None, + include_outside_points: bool = None, + ): + self.id = id + self.external_id = external_id + self.start = start + self.end = end + self.limit = limit + self.aggregates = aggregates + self.granularity = granularity + self.include_outside_points = include_outside_points diff --git a/cognite/client/data_classes/events.py b/cognite/client/data_classes/events.py new file mode 100644 index 0000000000..debb86d9c9 --- /dev/null +++ b/cognite/client/data_classes/events.py @@ -0,0 +1,181 @@ +from cognite.client.data_classes._base import * + + +# GenClass: Event +class Event(CogniteResource): + """An event represents something that happened at a given interval in time, e.g a failure, a work order etc. + + Args: + external_id (str): External Id provided by client. Should be unique within the project + start_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + end_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + type (str): Type of the event, e.g 'failure'. + subtype (str): Subtype of the event, e.g 'electrical'. + description (str): Textual description of the event. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): Asset IDs of related equipment that this event relates to. + source (str): The source of this event. + id (int): Javascript friendly internal ID given to the object. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + external_id: str = None, + start_time: int = None, + end_time: int = None, + type: str = None, + subtype: str = None, + description: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + id: int = None, + last_updated_time: int = None, + created_time: int = None, + cognite_client=None, + ): + self.external_id = external_id + self.start_time = start_time + self.end_time = end_time + self.type = type + self.subtype = subtype + self.description = description + self.metadata = metadata + self.asset_ids = asset_ids + self.source = source + self.id = id + self.last_updated_time = last_updated_time + self.created_time = created_time + self._cognite_client = cognite_client + + # GenStop + + +# GenClass: EventFilter +class EventFilter(CogniteFilter): + """Filter on events filter with exact match + + Args: + start_time (Dict[str, Any]): Range between two timestamps + end_time (Dict[str, Any]): Range between two timestamps + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): Asset IDs of related equipment that this event relates to. + source (str): The source of this event. + type (str): The event type + subtype (str): The event subtype + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + start_time: Dict[str, Any] = None, + end_time: Dict[str, Any] = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + type: str = None, + subtype: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + external_id_prefix: str = None, + cognite_client=None, + ): + self.start_time = start_time + self.end_time = end_time + self.metadata = metadata + self.asset_ids = asset_ids + self.source = source + self.type = type + self.subtype = subtype + self.created_time = created_time + self.last_updated_time = last_updated_time + self.external_id_prefix = external_id_prefix + self._cognite_client = cognite_client + + # GenStop + + +# GenUpdateClass: EventChange +class EventUpdate(CogniteUpdate): + """Changes will be applied to event. + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project + """ + + @property + def external_id(self): + return _PrimitiveEventUpdate(self, "externalId") + + @property + def start_time(self): + return _PrimitiveEventUpdate(self, "startTime") + + @property + def end_time(self): + return _PrimitiveEventUpdate(self, "endTime") + + @property + def description(self): + return _PrimitiveEventUpdate(self, "description") + + @property + def metadata(self): + return _ObjectEventUpdate(self, "metadata") + + @property + def asset_ids(self): + return _ListEventUpdate(self, "assetIds") + + @property + def source(self): + return _PrimitiveEventUpdate(self, "source") + + @property + def type(self): + return _PrimitiveEventUpdate(self, "type") + + @property + def subtype(self): + return _PrimitiveEventUpdate(self, "subtype") + + +class _PrimitiveEventUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> EventUpdate: + return self._set(value) + + +class _ObjectEventUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> EventUpdate: + return self._set(value) + + def add(self, value: Dict) -> EventUpdate: + return self._add(value) + + def remove(self, value: List) -> EventUpdate: + return self._remove(value) + + +class _ListEventUpdate(CogniteListUpdate): + def set(self, value: List) -> EventUpdate: + return self._set(value) + + def add(self, value: List) -> EventUpdate: + return self._add(value) + + def remove(self, value: List) -> EventUpdate: + return self._remove(value) + + # GenStop + + +class EventList(CogniteResourceList): + _RESOURCE = Event + _UPDATE = EventUpdate diff --git a/cognite/client/data_classes/files.py b/cognite/client/data_classes/files.py new file mode 100644 index 0000000000..3c751a5280 --- /dev/null +++ b/cognite/client/data_classes/files.py @@ -0,0 +1,158 @@ +from typing import Dict, List + +from cognite.client.data_classes._base import * + + +# GenClass: FilesMetadata +class FileMetadata(CogniteResource): + """No description. + + Args: + external_id (str): External Id provided by client. Should be unique within the project. + name (str): Name of the file. + source (str): The source of the file. + mime_type (str): File type. E.g. text/plain, application/pdf, .. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): No description. + id (int): Javascript friendly internal ID given to the object. + uploaded (bool): Whether or not the actual file is uploaded. This field is returned only by the API, it has no effect in a post body. + uploaded_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + external_id: str = None, + name: str = None, + source: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + id: int = None, + uploaded: bool = None, + uploaded_time: int = None, + created_time: int = None, + last_updated_time: int = None, + cognite_client=None, + ): + self.external_id = external_id + self.name = name + self.source = source + self.mime_type = mime_type + self.metadata = metadata + self.asset_ids = asset_ids + self.id = id + self.uploaded = uploaded + self.uploaded_time = uploaded_time + self.created_time = created_time + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + # GenStop + + +# GenClass: FilesSearchFilter.filter +class FileMetadataFilter(CogniteFilter): + """No description. + + Args: + name (str): Name of the file. + mime_type (str): File type. E.g. text/plain, application/pdf, .. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + asset_ids (List[int]): Only include files that reference these specific asset IDs. + source (str): The source of this event. + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + uploaded_time (Dict[str, Any]): Range between two timestamps + external_id_prefix (str): External Id provided by client. Should be unique within the project. + uploaded (bool): Whether or not the actual file is uploaded. This field is returned only by the API, it has no effect in a post body. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + mime_type: str = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + uploaded_time: Dict[str, Any] = None, + external_id_prefix: str = None, + uploaded: bool = None, + cognite_client=None, + ): + self.name = name + self.mime_type = mime_type + self.metadata = metadata + self.asset_ids = asset_ids + self.source = source + self.created_time = created_time + self.last_updated_time = last_updated_time + self.uploaded_time = uploaded_time + self.external_id_prefix = external_id_prefix + self.uploaded = uploaded + self._cognite_client = cognite_client + + # GenStop + + +# GenUpdateClass: FileChange +class FileMetadataUpdate(CogniteUpdate): + """Changes will be applied to file. + + Args: + """ + + @property + def external_id(self): + return _PrimitiveFileMetadataUpdate(self, "externalId") + + @property + def source(self): + return _PrimitiveFileMetadataUpdate(self, "source") + + @property + def metadata(self): + return _ObjectFileMetadataUpdate(self, "metadata") + + @property + def asset_ids(self): + return _ListFileMetadataUpdate(self, "assetIds") + + +class _PrimitiveFileMetadataUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> FileMetadataUpdate: + return self._set(value) + + +class _ObjectFileMetadataUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> FileMetadataUpdate: + return self._set(value) + + def add(self, value: Dict) -> FileMetadataUpdate: + return self._add(value) + + def remove(self, value: List) -> FileMetadataUpdate: + return self._remove(value) + + +class _ListFileMetadataUpdate(CogniteListUpdate): + def set(self, value: List) -> FileMetadataUpdate: + return self._set(value) + + def add(self, value: List) -> FileMetadataUpdate: + return self._add(value) + + def remove(self, value: List) -> FileMetadataUpdate: + return self._remove(value) + + # GenStop + + +class FileMetadataList(CogniteResourceList): + _RESOURCE = FileMetadata + _UPDATE = FileMetadataUpdate diff --git a/cognite/client/data_classes/iam.py b/cognite/client/data_classes/iam.py new file mode 100644 index 0000000000..118bb5ee0d --- /dev/null +++ b/cognite/client/data_classes/iam.py @@ -0,0 +1,138 @@ +from cognite.client.data_classes._base import * + + +# GenClass: ServiceAccount +class ServiceAccount(CogniteResource): + """No description. + + Args: + name (str): Unique name of the service account + groups (List[int]): List of group ids + id (int): No description. + is_deleted (bool): If this service account has been logically deleted + deleted_time (int): Time of deletion + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + groups: List[int] = None, + id: int = None, + is_deleted: bool = None, + deleted_time: int = None, + cognite_client=None, + ): + self.name = name + self.groups = groups + self.id = id + self.is_deleted = is_deleted + self.deleted_time = deleted_time + self._cognite_client = cognite_client + + # GenStop + + +class ServiceAccountList(CogniteResourceList): + _RESOURCE = ServiceAccount + _ASSERT_CLASSES = False + + +# GenClass: NewApiKeyResponseDTO +class APIKey(CogniteResource): + """No description. + + Args: + id (int): Internal id for the api key + service_account_id (int): id of the service account + created_time (int): Time of creating in unix ms + status (str): The status of the api key. + value (str): The api key to be used against the API + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + service_account_id: int = None, + created_time: int = None, + status: str = None, + value: str = None, + cognite_client=None, + ): + self.id = id + self.service_account_id = service_account_id + self.created_time = created_time + self.status = status + self.value = value + self._cognite_client = cognite_client + + # GenStop + + +class APIKeyList(CogniteResourceList): + _RESOURCE = APIKey + _ASSERT_CLASSES = False + + +# GenClass: Group +class Group(CogniteResource): + """No description. + + Args: + name (str): Name of the group + source_id (str): ID of the group in the source. If this is the same ID as a group in the IDP, a user in that group will implicitly be a part of this group as well. + capabilities (List[Dict[str, Any]]): No description. + id (int): No description. + is_deleted (bool): No description. + deleted_time (int): No description. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + source_id: str = None, + capabilities: List[Dict[str, Any]] = None, + id: int = None, + is_deleted: bool = None, + deleted_time: int = None, + cognite_client=None, + ): + self.name = name + self.source_id = source_id + self.capabilities = capabilities + self.id = id + self.is_deleted = is_deleted + self.deleted_time = deleted_time + self._cognite_client = cognite_client + + # GenStop + + +class GroupList(CogniteResourceList): + _RESOURCE = Group + _ASSERT_CLASSES = False + + +# GenClass: SecurityCategoryDTO +class SecurityCategory(CogniteResource): + """No description. + + Args: + name (str): Name of the security category + id (int): Id of the security category + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__(self, name: str = None, id: int = None, cognite_client=None): + self.name = name + self.id = id + self._cognite_client = cognite_client + + # GenStop + + +class SecurityCategoryList(CogniteResourceList): + _RESOURCE = SecurityCategory + _ASSERT_CLASSES = False diff --git a/cognite/client/data_classes/login.py b/cognite/client/data_classes/login.py new file mode 100644 index 0000000000..a33716b2f3 --- /dev/null +++ b/cognite/client/data_classes/login.py @@ -0,0 +1,30 @@ +from cognite.client.data_classes._base import CogniteResponse + + +class LoginStatus(CogniteResponse): + """Current login status + + Args: + user (str): Current user + logged_in (bool): Is user logged in + project (str): Current project + project_id (str): Current project id + """ + + def __init__(self, user: str, project: str, logged_in: bool, project_id: str, api_key_id: int): + self.user = user + self.project = project + self.project_id = project_id + self.logged_in = logged_in + self.api_key_id = api_key_id + + @classmethod + def _load(cls, api_response): + data = api_response["data"] + return cls( + user=data["user"], + project=data["project"], + logged_in=data["loggedIn"], + project_id=data["projectId"], + api_key_id=data.get("apiKeyId"), + ) diff --git a/cognite/client/data_classes/raw.py b/cognite/client/data_classes/raw.py new file mode 100644 index 0000000000..27b486a9b9 --- /dev/null +++ b/cognite/client/data_classes/raw.py @@ -0,0 +1,135 @@ +from collections import defaultdict + +from cognite.client.data_classes._base import * + + +# GenClass: RawDBRow +class Row(CogniteResource): + """No description. + + Args: + key (str): Unique row key + columns (Dict[str, Any]): Row data stored as a JSON object. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, key: str = None, columns: Dict[str, Any] = None, last_updated_time: int = None, cognite_client=None + ): + self.key = key + self.columns = columns + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + # GenStop + def to_pandas(self): + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The pandas DataFrame representing this instance. + """ + pd = utils.local_import("pandas") + return pd.DataFrame([self.columns], [self.key]) + + +class RowList(CogniteResourceList): + _RESOURCE = Row + _ASSERT_CLASSES = False + + def to_pandas(self): + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The pandas DataFrame representing this instance. + """ + pd = utils.local_import("pandas") + index = [row.key for row in self.data] + data = defaultdict(lambda: []) + for row in self.data: + for col_name, value in row.columns.items(): + data[col_name].append(value) + return pd.DataFrame(data, index) + + +# GenClass: RawDBTable +class Table(CogniteResource): + """A NoSQL database table to store customer data + + Args: + name (str): Unique name of the table + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__(self, name: str = None, cognite_client=None): + self.name = name + self._cognite_client = cognite_client + # GenStop + self._db_name = None + + def to_pandas(self): + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The pandas DataFrame representing this instance. + """ + return super().to_pandas([]) + + def rows(self, key: str = None, limit: int = None) -> Union[Row, RowList]: + """Get the rows in this table. + + Args: + key (str): Specify a key to return only that row. + limit (int): The number of rows to return. + + Returns: + Union[Row, RowList]: List of tables in this database. + """ + if key: + return self._cognite_client.raw.rows.retrieve(db_name=self._db_name, table_name=self.name, key=key) + return self._cognite_client.raw.rows.list(db_name=self._db_name, table_name=self.name, limit=limit) + + +class TableList(CogniteResourceList): + _RESOURCE = Table + _ASSERT_CLASSES = False + + +# GenClass: RawDB +class Database(CogniteResource): + """A NoSQL database to store customer data. + + Args: + name (str): Unique name of a database. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__(self, name: str = None, cognite_client=None): + self.name = name + self._cognite_client = cognite_client + + # GenStop + + def to_pandas(self): + """Convert the instance into a pandas DataFrame. + + Returns: + pandas.DataFrame: The pandas DataFrame representing this instance. + """ + return super().to_pandas([]) + + def tables(self, limit: int = None) -> TableList: + """Get the tables in this database. + + Args: + limit (int): The number of tables to return. + + Returns: + TableList: List of tables in this database. + """ + return self._cognite_client.raw.tables.list(db_name=self.name, limit=limit) + + +class DatabaseList(CogniteResourceList): + _RESOURCE = Database + _ASSERT_CLASSES = False diff --git a/cognite/client/data_classes/three_d.py b/cognite/client/data_classes/three_d.py new file mode 100644 index 0000000000..4e3787f498 --- /dev/null +++ b/cognite/client/data_classes/three_d.py @@ -0,0 +1,383 @@ +from cognite.client.data_classes._base import * + + +# GenClass: Model3D +class ThreeDModel(CogniteResource): + """No description. + + Args: + name (str): The name of the model. + id (int): The ID of the model. + created_time (int): The creation time of the resource, in milliseconds since January 1, 1970 at 00:00 UTC. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__(self, name: str = None, id: int = None, created_time: int = None, cognite_client=None): + self.name = name + self.id = id + self.created_time = created_time + self._cognite_client = cognite_client + + # GenStop + + +# GenUpdateClass: UpdateModel3D +class ThreeDModelUpdate(CogniteUpdate): + """No description. + + Args: + id (int): Javascript friendly internal ID given to the object. + """ + + @property + def name(self): + return _PrimitiveThreeDModelUpdate(self, "name") + + +class _PrimitiveThreeDModelUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> ThreeDModelUpdate: + return self._set(value) + + +class _ObjectThreeDModelUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> ThreeDModelUpdate: + return self._set(value) + + def add(self, value: Dict) -> ThreeDModelUpdate: + return self._add(value) + + def remove(self, value: List) -> ThreeDModelUpdate: + return self._remove(value) + + +class _ListThreeDModelUpdate(CogniteListUpdate): + def set(self, value: List) -> ThreeDModelUpdate: + return self._set(value) + + def add(self, value: List) -> ThreeDModelUpdate: + return self._add(value) + + def remove(self, value: List) -> ThreeDModelUpdate: + return self._remove(value) + + # GenStop + + +class ThreeDModelList(CogniteResourceList): + _RESOURCE = ThreeDModel + _UPDATE = ThreeDModelUpdate + + +# GenClass: Revision3D +class ThreeDModelRevision(CogniteResource): + """No description. + + Args: + id (int): The ID of the revision. + file_id (int): The file id. + published (bool): True if the revision is marked as published. + rotation (List[float]): No description. + camera (Dict[str, Any]): Initial camera position and target. + status (str): The status of the revision. + thumbnail_threed_file_id (int): The threed file ID of a thumbnail for the revision. Use /3d/files/{id} to retrieve the file. + thumbnail_url (str): The URL of a thumbnail for the revision. + asset_mapping_count (int): The number of asset mappings for this revision. + created_time (int): The creation time of the resource, in milliseconds since January 1, 1970 at 00:00 UTC. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + file_id: int = None, + published: bool = None, + rotation: List[float] = None, + camera: Dict[str, Any] = None, + status: str = None, + thumbnail_threed_file_id: int = None, + thumbnail_url: str = None, + asset_mapping_count: int = None, + created_time: int = None, + cognite_client=None, + ): + self.id = id + self.file_id = file_id + self.published = published + self.rotation = rotation + self.camera = camera + self.status = status + self.thumbnail_threed_file_id = thumbnail_threed_file_id + self.thumbnail_url = thumbnail_url + self.asset_mapping_count = asset_mapping_count + self.created_time = created_time + self._cognite_client = cognite_client + + # GenStop + + +# GenUpdateClass: UpdateRevision3D +class ThreeDModelRevisionUpdate(CogniteUpdate): + """No description. + + Args: + id (int): Javascript friendly internal ID given to the object. + """ + + @property + def published(self): + return _PrimitiveThreeDModelRevisionUpdate(self, "published") + + @property + def rotation(self): + return _ListThreeDModelRevisionUpdate(self, "rotation") + + @property + def camera(self): + return _ObjectThreeDModelRevisionUpdate(self, "camera") + + +class _PrimitiveThreeDModelRevisionUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> ThreeDModelRevisionUpdate: + return self._set(value) + + +class _ObjectThreeDModelRevisionUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> ThreeDModelRevisionUpdate: + return self._set(value) + + def add(self, value: Dict) -> ThreeDModelRevisionUpdate: + return self._add(value) + + def remove(self, value: List) -> ThreeDModelRevisionUpdate: + return self._remove(value) + + +class _ListThreeDModelRevisionUpdate(CogniteListUpdate): + def set(self, value: List) -> ThreeDModelRevisionUpdate: + return self._set(value) + + def add(self, value: List) -> ThreeDModelRevisionUpdate: + return self._add(value) + + def remove(self, value: List) -> ThreeDModelRevisionUpdate: + return self._remove(value) + + # GenStop + + +class ThreeDModelRevisionList(CogniteResourceList): + _RESOURCE = ThreeDModelRevision + _UPDATE = ThreeDModelRevisionUpdate + + +# GenClass: Node3D +class ThreeDNode(CogniteResource): + """No description. + + Args: + id (int): The ID of the node. + tree_index (int): The index of the node in the 3D model hierarchy, starting from 0. The tree is traversed in a depth-first order. + parent_id (int): The parent of the node, null if it is the root node. + depth (int): The depth of the node in the tree, starting from 0 at the root node. + name (str): The name of the node. + subtree_size (int): The number of descendants of the node, plus one (counting itself). + bounding_box (Dict[str, Any]): The bounding box of the subtree with this sector as the root sector. Is null if there are no geometries in the subtree. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + tree_index: int = None, + parent_id: int = None, + depth: int = None, + name: str = None, + subtree_size: int = None, + bounding_box: Dict[str, Any] = None, + cognite_client=None, + ): + self.id = id + self.tree_index = tree_index + self.parent_id = parent_id + self.depth = depth + self.name = name + self.subtree_size = subtree_size + self.bounding_box = bounding_box + self._cognite_client = cognite_client + + # GenStop + + +class ThreeDNodeList(CogniteResourceList): + _RESOURCE = ThreeDNode + _ASSERT_CLASSES = False + + +# GenClass: AssetMapping3D +class ThreeDAssetMapping(CogniteResource): + """No description. + + Args: + node_id (int): The ID of the node. + asset_id (int): The ID of the associated asset (Cognite's Assets API). + tree_index (int): A number describing the position of this node in the 3D hierarchy, starting from 0. The tree is traversed in a depth-first order. + subtree_size (int): The number of nodes in the subtree of this node (this number included the node itself). + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + node_id: int = None, + asset_id: int = None, + tree_index: int = None, + subtree_size: int = None, + cognite_client=None, + ): + self.node_id = node_id + self.asset_id = asset_id + self.tree_index = tree_index + self.subtree_size = subtree_size + self._cognite_client = cognite_client + + # GenStop + + +class ThreeDAssetMappingList(CogniteResourceList): + _RESOURCE = ThreeDAssetMapping + _ASSERT_CLASSES = False + + +# GenClass: RevealRevision3D +class ThreeDRevealRevision(CogniteResource): + """No description. + + Args: + id (int): The ID of the revision. + file_id (int): The file id. + published (bool): True if the revision is marked as published. + rotation (List[float]): No description. + camera (Dict[str, Any]): Initial camera position and target. + status (str): The status of the revision. + thumbnail_threed_file_id (int): The threed file ID of a thumbnail for the revision. Use /3d/files/{id} to retrieve the file. + thumbnail_url (str): The URL of a thumbnail for the revision. + asset_mapping_count (int): The number of asset mappings for this revision. + created_time (int): The creation time of the resource, in milliseconds since January 1, 1970 at 00:00 UTC. + scene_threed_files (List[Dict[str, Any]]): No description. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + file_id: int = None, + published: bool = None, + rotation: List[float] = None, + camera: Dict[str, Any] = None, + status: str = None, + thumbnail_threed_file_id: int = None, + thumbnail_url: str = None, + asset_mapping_count: int = None, + created_time: int = None, + scene_threed_files: List[Dict[str, Any]] = None, + cognite_client=None, + ): + self.id = id + self.file_id = file_id + self.published = published + self.rotation = rotation + self.camera = camera + self.status = status + self.thumbnail_threed_file_id = thumbnail_threed_file_id + self.thumbnail_url = thumbnail_url + self.asset_mapping_count = asset_mapping_count + self.created_time = created_time + self.scene_threed_files = scene_threed_files + self._cognite_client = cognite_client + + # GenStop + + +# GenClass: RevealNode3D +class ThreeDRevealNode(CogniteResource): + """No description. + + Args: + id (int): The ID of the node. + tree_index (int): The index of the node in the 3D model hierarchy, starting from 0. The tree is traversed in a depth-first order. + parent_id (int): The parent of the node, null if it is the root node. + depth (int): The depth of the node in the tree, starting from 0 at the root node. + name (str): The name of the node. + subtree_size (int): The number of descendants of the node, plus one (counting itself). + bounding_box (Dict[str, Any]): The bounding box of the subtree with this sector as the root sector. Is null if there are no geometries in the subtree. + sector_id (int): The sector the node is contained in. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + tree_index: int = None, + parent_id: int = None, + depth: int = None, + name: str = None, + subtree_size: int = None, + bounding_box: Dict[str, Any] = None, + sector_id: int = None, + cognite_client=None, + ): + self.id = id + self.tree_index = tree_index + self.parent_id = parent_id + self.depth = depth + self.name = name + self.subtree_size = subtree_size + self.bounding_box = bounding_box + self.sector_id = sector_id + self._cognite_client = cognite_client + + # GenStop + + +class ThreeDRevealNodeList(CogniteResourceList): + _RESOURCE = ThreeDRevealNode + _ASSERT_CLASSES = False + + +# GenClass: RevealSector3D +class ThreeDRevealSector(CogniteResource): + """No description. + + Args: + id (int): The id of the sector. + parent_id (int): The parent of the sector, null if it is the root sector. + path (str): String representing the path to the sector: 0/2/6/ etc. + depth (int): The depth of the sector in the sector tree, starting from 0 at the root sector. + bounding_box (Dict[str, Any]): The bounding box of the subtree with this sector as the root sector. Is null if there are no geometries in the subtree. + threed_files (List[Dict[str, Any]]): The file ID of the data file for this sector, with multiple versions supported. Use /3d/files/{id} to retrieve the file. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + parent_id: int = None, + path: str = None, + depth: int = None, + bounding_box: Dict[str, Any] = None, + threed_files: List[Dict[str, Any]] = None, + cognite_client=None, + ): + self.id = id + self.parent_id = parent_id + self.path = path + self.depth = depth + self.bounding_box = bounding_box + self.threed_files = threed_files + self._cognite_client = cognite_client + + # GenStop + + +class ThreeDRevealSectorList(CogniteResourceList): + _RESOURCE = ThreeDRevealSector + _ASSERT_CLASSES = False diff --git a/cognite/client/data_classes/time_series.py b/cognite/client/data_classes/time_series.py new file mode 100644 index 0000000000..ed9296cfdd --- /dev/null +++ b/cognite/client/data_classes/time_series.py @@ -0,0 +1,203 @@ +from typing import List + +from cognite.client.data_classes._base import * + + +# GenClass: GetTimeSeriesMetadataDTO +class TimeSeries(CogniteResource): + """No description. + + Args: + id (int): Generated id of the time series + external_id (str): Externally supplied id of the time series + name (str): Name of time series + is_string (bool): Whether the time series is string valued or not. + metadata (Dict[str, Any]): Additional metadata. String key -> String value. + unit (str): The physical unit of the time series. + asset_id (int): Asset that this time series belongs to. + is_step (bool): Whether the time series is a step series or not. + description (str): Description of the time series. + security_categories (List[int]): Security categories required in order to access this time series. + created_time (int): Time when this time-series is created in CDF in milliseconds since Jan 1, 1970. + last_updated_time (int): The latest time when this time-series is updated in CDF in milliseconds since Jan 1, 1970. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + id: int = None, + external_id: str = None, + name: str = None, + is_string: bool = None, + metadata: Dict[str, Any] = None, + unit: str = None, + asset_id: int = None, + is_step: bool = None, + description: str = None, + security_categories: List[int] = None, + created_time: int = None, + last_updated_time: int = None, + cognite_client=None, + ): + self.id = id + self.external_id = external_id + self.name = name + self.is_string = is_string + self.metadata = metadata + self.unit = unit + self.asset_id = asset_id + self.is_step = is_step + self.description = description + self.security_categories = security_categories + self.created_time = created_time + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + # GenStop + + def plot( + self, start="1d-ago", end="now", aggregates=None, granularity=None, id_labels: bool = False, *args, **kwargs + ): + plt = utils.local_import("matplotlib.pyplot") + identifier = utils.assert_at_least_one_of_id_or_external_id(self.id, self.external_id) + dps = self._cognite_client.datapoints.retrieve( + start=start, end=end, aggregates=aggregates, granularity=granularity, **identifier + ) + if id_labels: + dps.plot(*args, **kwargs) + else: + columns = {self.id: self.name} + for agg in aggregates or []: + columns["{}|{}".format(self.id, agg)] = "{}|{}".format(self.name, agg) + df = dps.to_pandas().rename(columns=columns) + df.plot(*args, **kwargs) + plt.show() + + +# GenClass: TimeSeriesSearchDTO.filter +class TimeSeriesFilter(CogniteFilter): + """Filtering parameters + + Args: + unit (str): Filter on unit (case-sensitive). + is_string (bool): Filter on isString. + is_step (bool): Filter on isStep. + metadata (Dict[str, Any]): Filter out timeseries that do not match these metadata fields and values (case-sensitive). Format is {"key1":"value1","key2":"value2"}. + asset_ids (List[int]): Filter out time series that are not linked to any of these assets. + created_time (Dict[str, Any]): Filter out time series with createdTime outside this range. + last_updated_time (Dict[str, Any]): Filter out time series with lastUpdatedTime outside this range. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + unit: str = None, + is_string: bool = None, + is_step: bool = None, + metadata: Dict[str, Any] = None, + asset_ids: List[int] = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + cognite_client=None, + ): + self.unit = unit + self.is_string = is_string + self.is_step = is_step + self.metadata = metadata + self.asset_ids = asset_ids + self.created_time = created_time + self.last_updated_time = last_updated_time + self._cognite_client = cognite_client + + # GenStop + + +# GenUpdateClass: TimeSeriesUpdate +class TimeSeriesUpdate(CogniteUpdate): + """Changes will be applied to timeseries. + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project. + """ + + @property + def external_id(self): + return _PrimitiveTimeSeriesUpdate(self, "externalId") + + @property + def name(self): + return _PrimitiveTimeSeriesUpdate(self, "name") + + @property + def metadata(self): + return _ObjectTimeSeriesUpdate(self, "metadata") + + @property + def unit(self): + return _PrimitiveTimeSeriesUpdate(self, "unit") + + @property + def asset_id(self): + return _PrimitiveTimeSeriesUpdate(self, "assetId") + + @property + def description(self): + return _PrimitiveTimeSeriesUpdate(self, "description") + + @property + def security_categories(self): + return _ListTimeSeriesUpdate(self, "securityCategories") + + +class _PrimitiveTimeSeriesUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> TimeSeriesUpdate: + return self._set(value) + + +class _ObjectTimeSeriesUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> TimeSeriesUpdate: + return self._set(value) + + def add(self, value: Dict) -> TimeSeriesUpdate: + return self._add(value) + + def remove(self, value: List) -> TimeSeriesUpdate: + return self._remove(value) + + +class _ListTimeSeriesUpdate(CogniteListUpdate): + def set(self, value: List) -> TimeSeriesUpdate: + return self._set(value) + + def add(self, value: List) -> TimeSeriesUpdate: + return self._add(value) + + def remove(self, value: List) -> TimeSeriesUpdate: + return self._remove(value) + + # GenStop + + +class TimeSeriesList(CogniteResourceList): + _RESOURCE = TimeSeries + _UPDATE = TimeSeriesUpdate + + def plot( + self, start="1d-ago", end="now", aggregates=None, granularity=None, id_labels: bool = False, *args, **kwargs + ): + plt = utils.local_import("matplotlib.pyplot") + dps = self._cognite_client.datapoints.retrieve( + id=[ts.id for ts in self.data], start=start, end=end, aggregates=aggregates, granularity=granularity + ) + if id_labels: + dps.plot(*args, **kwargs) + else: + columns = {} + for ts in self.data: + columns[ts.id] = ts.name + for agg in aggregates or []: + columns["{}|{}".format(ts.id, agg)] = "{}|{}".format(ts.name, agg) + df = dps.to_pandas().rename(columns=columns) + df.plot(*args, **kwargs) + plt.show() diff --git a/cognite/client/exceptions.py b/cognite/client/exceptions.py index e0bf136276..4acc1109f0 100644 --- a/cognite/client/exceptions.py +++ b/cognite/client/exceptions.py @@ -1,26 +1,33 @@ import json +from typing import * -class APIError(Exception): +class CogniteAPIError(Exception): """Cognite API Error - Raised if a given request fails. + Raised if a given request fails. If one or more of concurrent requests fails, this exception will also contain + information about which items were successfully processed (2xx), which may have been processed (5xx), and which have + failed to be processed (4xx). Args: - message (str): The error message produced by the API - code (int): The error code produced by the failure + message (str): The error message produced by the API + code (int): The error code produced by the failure x_request_id (str): The request-id generated for the failed request. - extra (Dict): A dict of any additional information. + extra (Dict): A dict of any additional information. + successful (List): List of items which were successfully proccessed. + failed (List): List of items which failed. + unknown (List): List of items which may or may not have been successfully processed. Examples: Catching an API-error and handling it based on the error code:: - from cognite.client import CogniteClient, APIError + from cognite.client import CogniteClient + from cognite.client.exceptions import CogniteAPIError - client = CogniteClient() + c = CogniteClient() try: - client.login.status() + c.login.status() except APIError as e: if e.code == 401: print("You are not authorized") @@ -31,16 +38,94 @@ class APIError(Exception): print("The message returned from the API: {}".format(e.message)) """ - def __init__(self, message, code=None, x_request_id=None, extra=None): + def __init__( + self, + message: str, + code: int = None, + x_request_id: str = None, + missing: List = None, + duplicated: List = None, + successful: List = None, + failed: List = None, + unknown: List = None, + unwrap_fn: Callable = None, + ): self.message = message self.code = code self.x_request_id = x_request_id - self.extra = extra + self.missing = missing + self.duplicated = duplicated + self.successful = successful or [] + self.failed = failed or [] + self.unknown = unknown or [] + self._unwrap_fn = unwrap_fn or (lambda x: x) def __str__(self): - if self.extra: - pretty_extra = json.dumps(self.extra, indent=4, sort_keys=True) - return "{} | code: {} | X-Request-ID: {}\n{}".format( - self.message, self.code, self.x_request_id, pretty_extra + msg = "{} | code: {} | X-Request-ID: {}".format(self.message, self.code, self.x_request_id) + if self.missing: + msg += "\nMissing: {}".format(self.missing) + if self.duplicated: + msg += "\nDuplicated: {}".format(self.duplicated) + if len(self.successful) > 0 or len(self.unknown) > 0 or len(self.failed) > 0: + msg += "\nThe API Failed to process some items.\nSuccessful (2xx): {}\nUnknown (5xx): {}\nFailed (4xx): {}".format( + [self._unwrap_fn(f) for f in self.successful], + [self._unwrap_fn(f) for f in self.unknown], + [self._unwrap_fn(f) for f in self.failed], ) - return "{} | code: {} | X-Request-ID: {}".format(self.message, self.code, self.x_request_id) + return msg + + +class CogniteNotFoundError(Exception): + """Cognite Not Found Error + + Raised if one or more of the requested ids/external ids are not found. + + Args: + not_found (List): The ids not found. + """ + + def __init__(self, not_found: List): + self.not_found = not_found + + def __str__(self): + return str(self.not_found) + + +class CogniteImportError(Exception): + """Cognite Import Error + + Raised if the user attempts to use functionality which requires an uninstalled package. + + Args: + module (str): Name of the module which could not be imported + message (str): The error message to output. + """ + + def __init__(self, module: str, message: str = None): + self.module = module + self.message = message or "The functionality your are trying to use requires '{}' to be installed.".format( + self.module + ) + + def __str__(self): + return self.message + + +class CogniteMissingClientError(Exception): + """Cognite Missing Client Error + + Raised if the user attempts to make use of a method which requires the cognite_client being set, but it is not. + """ + + def __str__(self): + return "A CogniteClient has not been set on this object. Pass it in the constructor to use it." + + +class CogniteAPIKeyError(Exception): + """Cognite API Key Error. + + Raised if an invalid API key has been used to authenticate. + """ + + def __str__(self): + return "Invalid API key." diff --git a/cognite/client/experimental/__init__.py b/cognite/client/experimental/__init__.py deleted file mode 100644 index 9e105bf9fa..0000000000 --- a/cognite/client/experimental/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -from cognite.client.experimental.model_hosting import ModelHostingClient -from cognite.client.experimental.datapoints import DatapointsClient -from cognite.client.experimental.sequences import SequencesClient -from cognite.client.experimental.time_series import TimeSeriesClient - - -class ExperimentalClient: - def __init__(self, client_factory): - self._client_factory = client_factory - self._model_hosting_client = ModelHostingClient(client_factory=self._client_factory) - self._datapoints_client = self._client_factory(DatapointsClient) - self._sequences_client = self._client_factory(SequencesClient) - self._time_series_client = self._client_factory(TimeSeriesClient) - - @property - def model_hosting(self) -> ModelHostingClient: - return self._model_hosting_client - - @property - def datapoints(self) -> DatapointsClient: - return self._datapoints_client - - @property - def sequences(self) -> SequencesClient: - return self._sequences_client - - @property - def time_series(self) -> TimeSeriesClient: - return self._time_series_client diff --git a/cognite/client/experimental/datapoints.py b/cognite/client/experimental/datapoints.py deleted file mode 100644 index 0bd6e59e06..0000000000 --- a/cognite/client/experimental/datapoints.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8 -*- -from concurrent.futures import ThreadPoolExecutor as Pool -from functools import partial -from typing import List - -import pandas as pd - -from cognite.client import _utils -from cognite.client._api_client import APIClient, CogniteResponse - - -class DatapointsResponse(CogniteResponse): - """Datapoints Response Object.""" - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"][0] - - def to_pandas(self): - """Returns data as a pandas dataframe""" - return pd.DataFrame(self.internal_representation["data"]["items"][0]["datapoints"]) - - -class DatapointsClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - self._LIMIT_AGG = 10000 - self._LIMIT = 100000 - - def get_datapoints(self, id, start, end=None, aggregates=None, granularity=None, **kwargs) -> DatapointsResponse: - """Returns a DatapointsObject containing a list of datapoints for the given query. - - This method will automate paging for the user and return all data for the given time period. - - Args: - id (int): The unique id of the timeseries to retrieve data for. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - Keyword Arguments: - include_outside_points (bool): No description - - workers (int): Number of download processes to run in parallell. Defaults to number returned by cpu_count(). - - limit (str): Max number of datapoints to return. If limit is specified, this method will not automate - paging and will return a maximum of 100,000 dps. - - Returns: - client.test_experimental.datapoints.DatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - start, end = _utils.interval_to_ms(start, end) - - if kwargs.get("limit"): - return self._get_datapoints_user_defined_limit( - id, - aggregates, - granularity, - start, - end, - limit=kwargs.get("limit"), - include_outside_points=kwargs.get("include_outside_points", False), - ) - - num_of_workers = kwargs.get("processes", self._num_of_workers) - if kwargs.get("include_outside_points") is True: - num_of_workers = 1 - - windows = _utils.get_datapoints_windows(start, end, granularity, num_of_workers) - - partial_get_dps = partial( - self._get_datapoints_helper_wrapper, - id=id, - aggregates=aggregates, - granularity=granularity, - include_outside_points=kwargs.get("include_outside_points", False), - ) - - with Pool(len(windows)) as p: - datapoints = p.map(partial_get_dps, windows) - - concat_dps = [] - [concat_dps.extend(el) for el in datapoints] - - return DatapointsResponse({"data": {"items": [{"id": id, "datapoints": concat_dps}]}}) - - def _get_datapoints_helper_wrapper(self, args, id, aggregates, granularity, include_outside_points): - return self._get_datapoints_helper( - id, aggregates, granularity, args["start"], args["end"], include_outside_points=include_outside_points - ) - - def _get_datapoints_helper(self, id, aggregates=None, granularity=None, start=None, end=None, **kwargs): - """Returns a list of datapoints for the given query. - - This method will automate paging for the given time period. - - Args: - id (int): The unique id of the timeseries to retrieve data for. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - Keyword Arguments: - include_outside_points (bool): No description. - - Returns: - list of datapoints: A list containing datapoint dicts. - """ - url = "/timeseries/{}/data".format(id) - - limit = self._LIMIT if aggregates is None else self._LIMIT_AGG - - params = { - "aggregates": aggregates, - "granularity": granularity, - "limit": limit, - "start": start, - "end": end, - "includeOutsidePoints": kwargs.get("include_outside_points", False), - } - - datapoints = [] - while (not datapoints or len(datapoints[-1]) == limit) and params["end"] > params["start"]: - res = self._get(url, params=params) - res = res.json()["data"]["items"][0]["datapoints"] - - if not res: - break - - datapoints.append(res) - latest_timestamp = int(datapoints[-1][-1]["timestamp"]) - params["start"] = latest_timestamp + (_utils.granularity_to_ms(granularity) if granularity else 1) - dps = [] - [dps.extend(el) for el in datapoints] - return dps - - def _get_datapoints_user_defined_limit( - self, id: int, aggregates: List, granularity: str, start, end, limit, **kwargs - ) -> DatapointsResponse: - """Returns a DatapointsResponse object with the requested data. - - No paging or parallelizing is done. - - Args: - id (int): The unique id of the timeseries to retrieve data for. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - limit (str): Max number of datapoints to return. Max is 100,000. - - Keyword Arguments: - include_outside_points (bool): No description. - - Returns: - client.test_experimental.datapoints.DatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/timeseries/{}/data".format(id) - - params = { - "aggregates": aggregates, - "granularity": granularity, - "limit": limit, - "start": start, - "end": end, - "includeOutsidePoints": kwargs.get("include_outside_points", False), - } - res = self._get(url, params=params) - res = res.json()["data"]["items"][0]["datapoints"] - return DatapointsResponse({"data": {"items": [{"id": id, "datapoints": res}]}}) diff --git a/cognite/client/experimental/model_hosting/__init__.py b/cognite/client/experimental/model_hosting/__init__.py deleted file mode 100644 index 175d391875..0000000000 --- a/cognite/client/experimental/model_hosting/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -from cognite.client.experimental.model_hosting.models import ModelsClient -from cognite.client.experimental.model_hosting.schedules import SchedulesClient -from cognite.client.experimental.model_hosting.source_packages import SourcePackageClient - - -class ModelHostingClient: - def __init__(self, client_factory): - self._client_factory = client_factory - self._models_client = self._client_factory(ModelsClient) - self._source_package_client = self._client_factory(SourcePackageClient) - self._schedules_client = self._client_factory(SchedulesClient) - - @property - def models(self) -> ModelsClient: - return self._models_client - - @property - def source_packages(self) -> SourcePackageClient: - return self._source_package_client - - @property - def schedules(self) -> SchedulesClient: - return self._schedules_client diff --git a/cognite/client/experimental/model_hosting/models.py b/cognite/client/experimental/model_hosting/models.py deleted file mode 100644 index 78432b0caa..0000000000 --- a/cognite/client/experimental/model_hosting/models.py +++ /dev/null @@ -1,530 +0,0 @@ -import os -from concurrent.futures.thread import ThreadPoolExecutor -from typing import Any, Dict, List - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResponse -from cognite.client.exceptions import APIError - - -class ModelResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item["id"] - self.name = item["name"] - self.project = item["project"] - self.description = item["description"] - self.created_time = item["createdTime"] - self.metadata = item["metadata"] - self.is_deprecated = item["isDeprecated"] - self.active_version_id = item["activeVersionId"] - self.input_fields = item["inputFields"] - self.output_fields = item["outputFields"] - - -class ModelCollectionResponse(CogniteCollectionResponse): - _RESPONSE_CLASS = ModelResponse - - -class ModelVersionResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item["id"] - self.is_deprecated = item["isDeprecated"] - self.training_details = item["trainingDetails"] - self.name = item["name"] - self.error_msg = item["errorMsg"] - self.model_id = item["modelId"] - self.created_time = item["createdTime"] - self.metadata = item["metadata"] - self.source_package_id = item["sourcePackageId"] - self.status = item["status"] - self.description = item["description"] - self.project = item["project"] - - -class ModelVersionCollectionResponse(CogniteCollectionResponse): - _RESPONSE_CLASS = ModelVersionResponse - - -class ModelArtifactResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.name = item["name"] - self.size = item["size"] - - -class ModelArtifactCollectionResponse(CogniteCollectionResponse): - _RESPONSE_CLASS = ModelArtifactResponse - - -class ModelLogResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.prediction_logs = item["predict"] - self.training_logs = item["train"] - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"] - - -class PredictionError(APIError): - pass - - -class ModelsClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - self._LIMIT = 1000 - - def create_model( - self, - name: str, - description: str = "", - metadata: Dict[str, Any] = None, - input_fields: List[Dict[str, str]] = None, - output_fields: List[Dict[str, str]] = None, - ) -> ModelResponse: - """Creates a new model - - Args: - name (str): Name of model - description (str): Description - metadata (Dict[str, Any]): Metadata about model - input_fields (List[str]): List of input fields the model accepts - output_fields (List[str]: List of output fields the model produces - - Returns: - experimental.model_hosting.models.ModelResponse: The created model. - """ - url = "/analytics/models" - model_body = { - "name": name, - "description": description, - "metadata": metadata or {}, - "inputFields": input_fields or [], - "outputFields": output_fields or [], - } - res = self._post(url, body=model_body) - return ModelResponse(res.json()) - - def list_models(self, limit: int = None, cursor: int = None, autopaging: bool = False) -> ModelCollectionResponse: - """List all models. - - Args: - limit (int): Maximum number of models to return. Defaults to 250. - cursor (str): Cursor to use to fetch next set of results. - autopaging (bool): Whether or not to automatically page through all results. Will disregard limit. - - Returns: - experimental.model_hosting.models.ModelCollectionResponse: List of models - """ - url = "/analytics/models" - params = {"cursor": cursor, "limit": limit if autopaging is False else self._LIMIT} - res = self._get(url, params=params, autopaging=autopaging) - return ModelCollectionResponse(res.json()) - - def get_model(self, id: int) -> ModelResponse: - """Get a model by id. - - Args: - id (int): Id of model to get. - - Returns: - experimental.model_hosting.models.ModelResponse: The requested model - """ - url = "/analytics/models/{}".format(id) - res = self._get(url) - return ModelResponse(res.json()) - - def update_model( - self, id: int, description: str = None, metadata: Dict[str, str] = None, active_version_id: int = None - ) -> ModelResponse: - """Update a model. - - Args: - id (int): Id of model to update. - description (str): Description of model. - metadata (Dict[str, str]): metadata about model. - active_version_id (int): Active version of model. - - Returns: - experimental.model_hosting.models.ModelResponse: Updated model - """ - url = "/analytics/models/{}/update".format(id) - body = {} - if description: - body.update({"description": {"set": description}}) - if metadata: - body.update({"metadata": {"set": metadata}}) - if active_version_id: - body.update({"activeVersionId": {"set": active_version_id}}) - res = self._put(url, body=body) - return ModelResponse(res.json()) - - def deprecate_model(self, id: int) -> ModelResponse: - """Deprecate a model. - - Args: - id (int): Id of model to deprecate. - - Returns: - experimental.model_hosting.models.ModelResponse: Deprecated model - """ - url = "/analytics/models/{}/deprecate".format(id) - res = self._put(url) - return ModelResponse(res.json()) - - def delete_model(self, id: int) -> None: - """Delete a model. - - Will also delete all versions and schedules for this model. - - Args: - id (int): Delete model with this id. - - Returns: - None - """ - url = "/analytics/models/{}".format(id) - self._delete(url) - - def create_model_version( - self, name: str, model_id: int, source_package_id: int, description: str = None, metadata: Dict = None - ) -> ModelVersionResponse: - """Create a model version without deploying it. - - Then you can optionally upload artifacts to the model version and later deploy it. - - Args: - name (str): Name of the the model version. - model_id (int): Create the version on the model with this id. - source_package_id (int): Use the source package with this id. The source package must have an available - predict operation. - description (str): Description of model version - metadata (Dict[str, Any]): Metadata about model version - - Returns: - ModelVersionResponse: The created model version. - """ - url = "/analytics/models/{}/versions".format(model_id) - body = { - "name": name, - "description": description or "", - "sourcePackageId": source_package_id, - "metadata": metadata or {}, - } - res = self._post(url, body=body) - return ModelVersionResponse(res.json()) - - def deploy_awaiting_model_version(self, model_id: int, version_id: int) -> ModelVersionResponse: - """Deploy an already created model version awaiting manual deployment. - - The model version must have status AWAITING_MANUAL_DEPLOYMENT in order for this to work. - - Args: - model_id (int): The id of the model containing the version to deploy. - version_id (int): The id of the model version to deploy. - Returns: - ModelVersionResponse: The deployed model version. - """ - url = "/analytics/models/{}/versions/{}/deploy".format(model_id, version_id) - res = self._post(url, body={}) - return ModelVersionResponse(res.json()) - - def deploy_model_version( - self, - name: str, - model_id: int, - source_package_id: int, - artifacts_directory: str = None, - description: str = None, - metadata: Dict = None, - ) -> ModelVersionResponse: - """This will create and deploy a model version. - - If artifacts_directory is specified, it will traverse that directory recursively and - upload all artifacts in that directory before deploying. - - Args: - name (str): Name of the the model version. - model_id (int): Create the version on the model with this id. - source_package_id (int): Use the source package with this id. The source package must have an available - predict operation. - artifacts_directory (str, optional): Absolute path of directory containing artifacts. - description (str, optional): Description of model version - metadata (Dict[str, Any], optional): Metadata about model version - - Returns: - ModelVersionResponse: The created model version. - """ - model_version = self.create_model_version(name, model_id, source_package_id, description, metadata) - if artifacts_directory: - self.upload_artifacts_from_directory(model_id, model_version.id, directory=artifacts_directory) - return self.deploy_awaiting_model_version(model_id, model_version.id) - - def train_and_deploy_model_version( - self, - name: str, - model_id: int, - source_package_id: int, - train_source_package_id: int = None, - metadata: Dict = None, - description: str = None, - args: Dict[str, Any] = None, - scale_tier: str = None, - machine_type: str = None, - ) -> ModelVersionResponse: - """Train and deploy a new version of a model. - - This will instantiate a training job and automatically deploy the model upon completion. - - Args: - model_id (int): Create a new version under the model with this id - name (str): Name of model version. - source_package_id (int): Use the source package with this id - train_source_package_id (int): Use this source package for training. If omitted, will default to - source_package_id. - metadata (Dict[str, Any]): Metadata about model version - description (str): Description of model version - args (Dict[str, Any]): Dictionary of arguments to pass to the training job. - scale_tier (str): Which scale tier to use. Must be either "BASIC" or "CUSTOM". - machine_type (str): Specify a machine type. Applies only if scale_tier is "CUSTOM". - - Returns: - experimental.model_hosting.models.ModelVersionResponse: The created model version. - """ - url = "/analytics/models/{}/versions/train".format(model_id) - if args and "data_spec" in args: - data_spec = args["data_spec"] - if hasattr(data_spec, "dump"): - args["data_spec"] = data_spec.dump() - body = { - "name": name, - "description": description or "", - "sourcePackageId": source_package_id, - "trainingDetails": { - "sourcePackageId": train_source_package_id or source_package_id, - "args": args or {}, - "scaleTier": scale_tier or "BASIC", - "machineType": machine_type, - }, - "metadata": metadata or {}, - } - res = self._post(url, body=body) - return ModelVersionResponse(res.json()) - - def list_model_versions( - self, model_id: int, limit: int = None, cursor: str = None, autopaging: bool = False - ) -> ModelVersionCollectionResponse: - """Get all versions of a specific model. - - Args: - model_id (int): Get versions for the model with this id. - limit (int): Maximum number of model versions to return. Defaults to 250. - cursor (str): Cursor to use to fetch next set of results. - autopaging (bool): Whether or not to automatically page through all results. Will disregard limit. - - Returns: - experimental.model_hosting.models.ModelVersionCollectionResponse: List of model versions - """ - url = "/analytics/models/{}/versions".format(model_id) - params = {"cursor": cursor, "limit": limit if autopaging is False else self._LIMIT} - res = self._get(url, params=params, autopaging=True) - return ModelVersionCollectionResponse(res.json()) - - def get_model_version(self, model_id: int, version_id: int) -> ModelVersionResponse: - """Get a specific model version by id. - - Args: - model_id (int): Id of model which has the model version. - version_id (int): Id of model version. - - Returns: - experimental.model_hosting.models.ModelVersionResponse: The requested model version - """ - url = "/analytics/models/{}/versions/{}".format(model_id, version_id) - res = self._get(url) - return ModelVersionResponse(res.json()) - - def update_model_version( - self, model_id: int, version_id: int, description: str = None, metadata: Dict[str, str] = None - ) -> ModelVersionResponse: - """Update description or metadata on a model version. - - Args: - model_id (int): Id of model containing the model version. - version_id (int): Id of model version to update. - description (str): New description. - metadata (Dict[str, str]): New metadata - - Returns: - ModelVersionResponse: The updated model version. - """ - url = "/analytics/models/{}/versions/{}/update".format(model_id, version_id) - body = {} - if description: - body.update({"description": {"set": description}}) - if metadata: - body.update({"metadata": {"set": metadata}}) - - res = self._put(url, body=body) - return ModelVersionResponse(res.json()) - - def deprecate_model_version(self, model_id: int, version_id: int) -> ModelVersionResponse: - """Deprecate a model version - - Args: - model_id (int): Id of model - version_id (int): Id of model version to deprecate - - Returns: - ModelVersionResponse: The deprecated model version - """ - url = "/analytics/models/{}/versions/{}/deprecate".format(model_id, version_id) - res = self._put(url) - return ModelVersionResponse(res.json()) - - def delete_model_version(self, model_id: int, version_id: int) -> None: - """Delete a model version by id. - - Args: - model_id (int): Id of model which has the model version. - version_id (int): Id of model version. - - Returns: - None - """ - url = "/analytics/models/{}/versions/{}".format(model_id, version_id) - self._delete(url) - - def online_predict( - self, model_id: int, version_id: int = None, instances: List = None, args: Dict[str, Any] = None - ) -> List: - """Perform online prediction on a models active version or a specified version. - - Args: - model_id (int): Perform a prediction on the model with this id. Will use active version. - version_id (int): Use this version instead of the active version. (optional) - instances (List): List of JSON serializable instances to pass to your model one-by-one. - args (Dict[str, Any]) Dictinoary of keyword arguments to pass to your predict method. - - Returns: - List: List of predictions for each instance. - """ - url = "/analytics/models/{}/predict".format(model_id) - if instances: - for i, instance in enumerate(instances): - if hasattr(instance, "dump"): - instances[i] = instance.dump() - if version_id: - url = "/analytics/models/{}/versions/{}/predict".format(model_id, version_id) - body = {"instances": instances, "args": args or {}} - res = self._put(url, body=body).json() - if "error" in res: - raise PredictionError(message=res["error"]["message"], code=res["error"]["code"]) - return res["data"]["predictions"] - - def list_artifacts(self, model_id: int, version_id: int) -> ModelArtifactCollectionResponse: - """List the artifacts associated with the specified model version. - - Args: - model_id (int): Id of model - version_id: Id of model version to get artifacts for - - Returns: - experimental.model_hosting.models.ModelArtifactCollectionResponse: List of artifacts - """ - url = "/analytics/models/{}/versions/{}/artifacts".format(model_id, version_id) - res = self._get(url) - return ModelArtifactCollectionResponse(res.json()) - - def download_artifact(self, model_id: int, version_id: int, name: str, directory: str = None) -> None: - """Download an artifact to a directory. Defaults to current working directory. - - Args: - model_id (int): Id of model - version_id (int): Id of model version. - name (int): Name of artifact. - directory (int): Directory to place artifact in. Defaults to current working directory. - - Returns: - None - """ - directory = directory or os.getcwd() - file_path = os.path.join(directory, name) - - url = "/analytics/models/{}/versions/{}/artifacts/{}".format(model_id, version_id, name) - download_url = self._get(url).json()["data"]["downloadUrl"] - with open(file_path, "wb") as fh: - response = self._request_session.get(download_url).content - fh.write(response) - - def upload_artifact_from_file(self, model_id: int, version_id: int, name: str, file_path: str) -> None: - """Upload an artifact to a model version. - - The model version must have status AWAITING_MANUAL_DEPLOYMENT in order for this to work. - - Args: - model_id (int): The id of the model. - version_id (int): The id of the model version to upload the artifacts to. - name (str): The name of the artifact. - file_path (str): The local path of the artifact. - Returns: - None - """ - url = "/analytics/models/{}/versions/{}/artifacts/upload".format(model_id, version_id) - body = {"name": name} - res = self._post(url, body=body) - upload_url = res.json()["data"]["uploadUrl"] - self._upload_file(upload_url, file_path) - - def upload_artifacts_from_directory(self, model_id: int, version_id: int, directory: str) -> None: - """Upload all files in directory recursively. - - Args: - model_id (int): The id of the model. - version_id (int): The id of the model version to upload the artifacts to. - directory (int): Absolute path of directory to upload artifacts from. - Returns: - None - """ - upload_tasks = [] - for root, dirs, files in os.walk(directory): - for file_name in files: - file_path = os.path.join(root, file_name) - full_file_name = os.path.relpath(file_path, directory) - upload_tasks.append((model_id, version_id, full_file_name, file_path)) - self._execute_tasks_concurrently(self.upload_artifact_from_file, upload_tasks) - - @staticmethod - def _execute_tasks_concurrently(func, tasks): - with ThreadPoolExecutor(16) as p: - futures = [p.submit(func, *task) for task in tasks] - return [future.result() for future in futures] - - def _upload_file(self, upload_url, file_path): - with open(file_path, "rb") as fh: - mydata = fh.read() - response = self._request_session.put(upload_url, data=mydata) - return response - - def get_logs(self, model_id: int, version_id: int, log_type: str = None) -> ModelLogResponse: - """Get logs for prediction and/or training routine of a specific model version. - - Args: - model_id (int): Id of model. - version_id (int): Id of model version to get logs for. - log_type (str): Which routine to get logs from. Must be 'train’, 'predict’, or ‘both’. Defaults to 'both'. - - Returns: - ModelLogResponse: An object containing the requested logs. - """ - url = "/analytics/models/{}/versions/{}/log".format(model_id, version_id) - params = {"logType": log_type} - res = self._get(url, params=params) - return ModelLogResponse(res.json()) diff --git a/cognite/client/experimental/model_hosting/schedules.py b/cognite/client/experimental/model_hosting/schedules.py deleted file mode 100644 index 44cd97ec62..0000000000 --- a/cognite/client/experimental/model_hosting/schedules.py +++ /dev/null @@ -1,122 +0,0 @@ -from typing import Any, Dict, List, Union - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResponse - - -class ScheduleResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.is_deprecated = item["isDeprecated"] - self.name = item["name"] - self.schedule_data_spec = item["dataSpec"] - self.model_id = item["modelId"] - self.created_time = item["createdTime"] - self.metadata = item["metadata"] - self.id = item["id"] - self.args = item["args"] - self.description = item["description"] - - -class ScheduleCollectionResponse(CogniteCollectionResponse): - _RESPONSE_CLASS = ScheduleResponse - - -class SchedulesClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - self._LIMIT = 1000 - - def create_schedule( - self, - model_id: int, - name: str, - schedule_data_spec: Any, - description: str = None, - args: Dict = None, - metadata: Dict = None, - ) -> ScheduleResponse: - """Create a new schedule on a given model. - - Args: - model_id (int): Id of model to create schedule on - name (str): Name of schedule - schedule_data_spec (Any): Specification of schedule input/output. Can be either a dictionary or a - ScheduleDataSpec object from cognite-data-fetcher - description (str): Description for schedule - args (Dict): Dictionary of keyword arguments to pass to predict method. - metadata (Dict): Dictionary of metadata about schedule - - Returns: - experimental.model_hosting.schedules.ScheduleResponse: The created schedule. - """ - url = "/analytics/models/schedules" - - if hasattr(schedule_data_spec, "dump"): - schedule_data_spec = schedule_data_spec.dump() - - body = { - "name": name, - "description": description, - "modelId": model_id, - "args": args or {}, - "dataSpec": schedule_data_spec, - "metadata": metadata or {}, - } - res = self._post(url, body=body) - return ScheduleResponse(res.json()) - - def list_schedules( - self, limit: int = None, cursor: int = None, autopaging: bool = False - ) -> ScheduleCollectionResponse: - """Get all schedules. - - Args: - limit (int): Maximum number of schedules to return. Defaults to 250. - cursor (str): Cursor to use to fetch next set of results. - autopaging (bool): Whether or not to automatically page through all results. Will disregard limit. - - Returns: - experimental.model_hosting.schedules.ScheduleCollectionResponse: The requested schedules. - """ - url = "/analytics/models/schedules" - params = {"cursor": cursor, "limit": limit if autopaging is False else self._LIMIT} - res = self._get(url, params=params, autopaging=autopaging) - return ScheduleCollectionResponse(res.json()) - - def get_schedule(self, id: int) -> ScheduleResponse: - """Get a schedule by id. - - Args: - id (int): Id of schedule to get. - Returns: - experimental.model_hosting.schedules.ScheduleResponse: The requested schedule. - """ - url = "/analytics/models/schedules/{}".format(id) - res = self._get(url=url) - return ScheduleResponse(res.json()) - - def deprecate_schedule(self, id: int) -> ScheduleResponse: - """Deprecate a schedule. - - Args: - id (int): Id of schedule to deprecate - - Returns: - experimental.model_hosting.schedules.ScheduleResponse: The deprecated schedule. - """ - url = "/analytics/models/schedules/{}/deprecate".format(id) - res = self._put(url) - return ScheduleResponse(res.json()) - - def delete_schedule(self, id: int) -> None: - """Delete a schedule by id. - - Args: - id (int): The id of the schedule to delete. - - Returns: - None - """ - url = "/analytics/models/schedules/{}".format(id) - self._delete(url=url) diff --git a/cognite/client/experimental/model_hosting/source_packages.py b/cognite/client/experimental/model_hosting/source_packages.py deleted file mode 100644 index 86b39e4940..0000000000 --- a/cognite/client/experimental/model_hosting/source_packages.py +++ /dev/null @@ -1,258 +0,0 @@ -import os -import pkgutil -import re -from collections import namedtuple -from subprocess import check_call -from typing import Dict, List, NamedTuple, Tuple - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResponse - - -class CreateSourcePackageResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item["id"] - self.upload_url = item.get("uploadUrl") - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"] - - -class SourcePackageResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.is_deprecated = item["isDeprecated"] - self.package_name = item["packageName"] - self.is_uploaded = item["isUploaded"] - self.name = item["name"] - self.available_operations = item["availableOperations"] - self.created_time = item["createdTime"] - self.runtime_version = item["runtimeVersion"] - self.id = item["id"] - self.metadata = item["metadata"] - self.description = item["description"] - self.project = item["project"] - - -class SourcePackageCollectionResponse(CogniteCollectionResponse): - _RESPONSE_CLASS = SourcePackageResponse - - -class SourcePackageClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - self._LIMIT = 1000 - - def upload_source_package( - self, - name: str, - package_name: str, - available_operations: List[str], - runtime_version: str, - description: str = None, - metadata: Dict = None, - file_path: str = None, - ) -> CreateSourcePackageResponse: - """Upload a source package to the model hosting environment. - - Args: - name (str): Name of source package - package_name (str): name of root package for model - available_operations (List[str]): List of routines which this source package supports ["predict", "train"] - runtime_version (str): Version of environment in which the source-package should run. Currently only 0.1. - description (str): Description for source package - metadata (Dict): User defined key value pair of additional information. - file_path (str): File path of source package distribution. If not specified, a download url will be returned. - - Returns: - experimental.model_hosting.source_packages.CreateSourcePackageResponse: An response object containing Source package ID - if file_path was specified. Else, both source package id and upload url. - - """ - url = "/analytics/models/sourcepackages" - body = { - "name": name, - "description": description or "", - "packageName": package_name, - "availableOperations": available_operations, - "metadata": metadata or {}, - "runtimeVersion": runtime_version, - } - res = self._post(url, body=body).json() - if file_path: - self._upload_file(res["data"]["uploadUrl"], file_path) - del res["data"]["uploadUrl"] - return CreateSourcePackageResponse(res) - return CreateSourcePackageResponse(res) - - def _get_model_py_files(self, path) -> List: - files_containing_model_py = [] - for root, dirs, files in os.walk(path): - if "model.py" in files: - package_name = os.path.basename(root) - file_path = os.path.join(root, "model.py") - files_containing_model_py.append((package_name, file_path)) - return files_containing_model_py - - def _find_model_file_and_extract_details(self, package_directory: str) -> Tuple: - num_of_eligible_model_py_files = 0 - for package_name, file_path in self._get_model_py_files(package_directory): - with open(file_path, "r") as f: - file_content = f.read() - if re.search("class Model", file_content): - num_of_eligible_model_py_files += 1 - model_package_name = package_name - model_file_content = file_content - - assert num_of_eligible_model_py_files != 0, "Could not locate a file named model.py containing a Model class" - assert num_of_eligible_model_py_files == 1, "Multiple model.py files with a Model class in your source package" - - available_operations = [] - if re.search("def train\(", model_file_content): - available_operations.append("TRAIN") - if re.search("def predict\(", model_file_content): - if re.search("def load\(", model_file_content): - available_operations.append("PREDICT") - else: - raise AssertionError("Your Model class defines predict() but not load().") - assert len(available_operations) > 0, "Your model does not define a train or a predict method" - return model_package_name, available_operations - - def _build_distribution(self, package_path) -> str: - check_call("cd {} && python setup.py sdist".format(package_path), shell=True) - dist_directory = os.path.join(package_path, "dist") - for file in os.listdir(dist_directory): - if file.endswith(".tar.gz"): - return os.path.join(dist_directory, file) - - def build_and_upload_source_package( - self, name: str, runtime_version: str, package_directory: str, description: str = None, metadata: Dict = None - ) -> CreateSourcePackageResponse: - """Build a distribution for a source package and upload it to the model hosting environment. - - This method will recursively search through your package and infer available_operations as well as the package - name. - - Args: - name (str): Name of source package - runtime_version (str): Version of environment in which the source-package should run. Currently only 0.1. - description (str): Description for source package - metadata (Dict): User defined key value pair of additional information. - package_directory (str): Absolute path of directory containing your setup.py file. - - Returns: - experimental.model_hosting.source_packages.CreateSourcePackageResponse: An response object containing Source package ID - if file_path was specified. Else, both source package id and upload url. - """ - package_name, available_operations = self._find_model_file_and_extract_details(package_directory) - tar_gz_path = self._build_distribution(package_directory) - - return self.upload_source_package( - name=name, - package_name=package_name, - available_operations=available_operations, - runtime_version=runtime_version, - description=description, - metadata=metadata, - file_path=tar_gz_path, - ) - - def _upload_file(self, upload_url, file_path): - with open(file_path, "rb") as fh: - mydata = fh.read() - response = self._request_session.put(upload_url, data=mydata) - return response - - def list_source_packages( - self, limit: int = None, cursor: str = None, autopaging: bool = False - ) -> SourcePackageCollectionResponse: - """List all model source packages. - - Args: - limit (int): Maximum number of source_packages to return. Defaults to 250. - cursor (str): Cursor to use to fetch next set of results. - autopaging (bool): Whether or not to automatically page through all results. Will disregard limit. - - Returns: - experimental.model_hosting.source_packages.SourcePackageCollectionResponse: List of source packages. - """ - url = "/analytics/models/sourcepackages" - params = {"cursor": cursor, "limit": limit if autopaging is False else self._LIMIT} - res = self._get(url, params=params, autopaging=autopaging) - return SourcePackageCollectionResponse(res.json()) - - def get_source_package(self, id: int) -> SourcePackageResponse: - """Get source package by id. - - Args: - id (int): Id of soure package to get. - - Returns: - experimental.model_hosting.source_packages.SourcePackageResponse: The requested source package. - """ - url = "/analytics/models/sourcepackages/{}".format(id) - res = self._get(url) - return SourcePackageResponse(res.json()) - - def delete_source_package(self, id: int) -> None: - """Delete source package by id. - - Args: - id (int): Id of soure package to delete. - - Returns: - None - """ - url = "/analytics/models/sourcepackages/{}".format(id) - self._delete(url) - - def deprecate_source_package(self, id: int) -> SourcePackageResponse: - """Deprecate a source package by id. - - Args: - id (int): Id of soure package to get. - - Returns: - experimental.model_hosting.source_packages.SourcePackageResponse: The requested source package. - """ - url = "/analytics/models/sourcepackages/{}/deprecate".format(id) - res = self._put(url) - return SourcePackageResponse(res.json()) - - def download_source_package_code(self, id: int, directory: str = None) -> None: - """Download the tarball for a source package to a specified directory. - - - Args: - id (int): Id of source package. - directory (str): Directory to put source package in. Defaults to current working directory. - - Returns: - None - """ - directory = directory or os.getcwd() - file_path = os.path.join(directory, self.get_source_package(id).name + ".tar.gz") - url = "/analytics/models/sourcepackages/{}/code".format(id) - download_url = self._get(url).json()["data"]["downloadUrl"] - with open(file_path, "wb") as fh: - response = self._request_session.get(download_url).content - fh.write(response) - - def delete_source_package_code(self, id: int) -> None: - """Delete the code/tarball for the source package from the cloud storage location. - This will only work if the source package has been deprecated. - - Warning: This cannot be undone. - - Args: - id (int): Id of the source package. - - Returns: - None - - """ - url = "/analytics/models/sourcepackages/{}/code".format(id) - self._delete(url) diff --git a/cognite/client/experimental/sequences.py b/cognite/client/experimental/sequences.py deleted file mode 100644 index fddcaa73a6..0000000000 --- a/cognite/client/experimental/sequences.py +++ /dev/null @@ -1,323 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from typing import List - -import pandas as pd - -from cognite.client._api_client import APIClient - - -class Column: - """Data transfer object for a column. - - Args: - id (int): ID of the column. - name (str): Name of the column. - external_id (str): External ID of the column. - value_type (str): Data type of the column. - metadata (dict): Custom, application specific metadata. String key -> String Value. - """ - - def __init__( - self, id: int = None, name: str = None, external_id: str = None, value_type: str = None, metadata: dict = None - ): - if value_type is None: - raise ValueError("value_type must not be None") - self.id = id - self.name = name - self.externalId = external_id - self.valueType = value_type - self.metadata = metadata - - @staticmethod - def from_JSON(the_column: dict): - return Column( - id=the_column["id"], - name=the_column["name"], - external_id=the_column.get("externalId", None), - value_type=the_column["valueType"], - metadata=the_column["metadata"], - ) - - -class Sequence: - """Data transfer object for a sequence. - - Args: - id (int): ID of the sequence. - name (str): Name of the sequence. - external_id (str): External ID of the sequence. - asset_id (int): ID of the asset the sequence is connected to, if any. - columns (List[Column]): List of columns in the sequence. - description (str): Description of the sequence. - metadata (dict): Custom, application specific metadata. String key -> String Value. - - """ - - def __init__( - self, - id: int = None, - name: str = None, - external_id: str = None, - asset_id: int = None, - columns: List[Column] = None, - description: str = None, - metadata: dict = None, - ): - if columns is None: - raise ValueError("columns must not be None") - self.id = id - self.name = name - self.externalId = external_id - self.assetId = asset_id - self.columns = columns - self.description = description - self.metadata = metadata - - @staticmethod - def from_JSON(the_sequence: dict): - return Sequence( - id=the_sequence["id"], - name=the_sequence.get("name", None), - external_id=the_sequence.get("externalId", None), - asset_id=the_sequence.get("assetId", None), - columns=[Column.from_JSON(the_column) for the_column in the_sequence["columns"]], - description=the_sequence["description"], - metadata=the_sequence["metadata"], - ) - - -class RowValue: - """Data transfer object for the value in a row in a sequence. - - Args: - column_id (int): The ID of the column that this value is for. - value (str): The actual value. - """ - - def __init__(self, column_id: int, value: str): - self.columnId = column_id - self.value = value - - @staticmethod - def from_JSON(the_row_value: dict): - return RowValue(column_id=the_row_value["columnId"], value=the_row_value["value"]) - - -class Row: - """Data transfer object for a row of data in a sequence. - - Args: - row_number (int): The row number for this row. - values (list): The values in this row. - """ - - def __init__(self, row_number: int, values: List[RowValue]): - self.rowNumber = row_number - self.values = values - - @staticmethod - def from_JSON(the_row: dict): - return Row( - row_number=the_row["rowNumber"], - values=[RowValue.from_JSON(the_row_value) for the_row_value in the_row["values"]], - ) - - def get_row_as_csv(self): - return ",".join([str(x.value) for x in self.values]) - - -class SequenceDataResponse: - """Data transfer object for the data in a sequence, used when receiving data. - - Args: - rows (list): List of rows with the data. - """ - - def __init__(self, rows: List[Row]): - self.rows = rows - - @staticmethod - def from_JSON(the_data: dict): - return SequenceDataResponse(rows=[Row.from_JSON(the_row) for the_row in the_data["rows"]]) - - @staticmethod - def _row_has_value_for_column(row: Row, column_id: int): - return column_id in [value.columnId for value in row.values] - - @staticmethod - def _get_value_for_column(row: Row, column_id: int): - return next(value.value for value in row.values if value.columnId == column_id) - - def to_pandas(self): - """Returns data as a pandas dataframe""" - - # Create the empty dataframe - column_ids = [value.columnId for value in self.rows[0].values] - my_df = pd.DataFrame(columns=column_ids) - # Fill the dataframe with values. We might not have data for every column, so we need to be careful - for row in self.rows: - data_this_row = [] - for column_id in column_ids: - # Do we have a value for this column? - if self._row_has_value_for_column(row, column_id): - data_this_row.append(self._get_value_for_column(row, column_id)) - else: - data_this_row.append("null") - my_df.loc[len(my_df)] = data_this_row - return my_df - - def to_json(self): - """Returns data as a json object""" - raise NotImplementedError - - -class SequenceDataRequest: - """Data transfer object for requesting sequence data. - - Args: - inclusive_from (int): Row number to get from (inclusive). - inclusive_to (int): Row number to get to (inclusive). - limit (int): How many rows to return. - column_ids (List[int]): ids of the columns to get data for. - """ - - def __init__(self, inclusive_from: int, inclusive_to: int, limit: int = 100, column_ids: List[int] = None): - self.inclusiveFrom = inclusive_from - self.inclusiveTo = inclusive_to - self.limit = limit - self.columnIds = column_ids or [] - - -class SequencesClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - - def post_sequences(self, sequences: List[Sequence]) -> Sequence: - """Create a new time series. - - Args: - sequences (list[test_experimental.dto.Sequence]): List of sequence data transfer objects to create. - - Returns: - client.test_experimental.sequences.Sequence: The created sequence - """ - url = "/sequences" - - # Remove the id field from the sequences to be posted, as including them will lead to 400's since sequences that - # are not created yet should not have id's yet. - for sequence in sequences: - del sequence.id - for column in sequence.columns: - del column.id - - body = {"items": [sequence.__dict__ for sequence in sequences]} - res = self._post(url, body=body) - json_response = json.loads(res.text) - the_sequence = json_response["data"]["items"][0] - return Sequence.from_JSON(the_sequence) - - def get_sequence_by_id(self, id: int) -> Sequence: - """Returns a Sequence object containing the requested sequence. - - Args: - id (int): ID of the sequence to look up - - Returns: - client.test_experimental.sequences.Sequence: A data object containing the requested sequence. - """ - url = "/sequences/{}".format(id) - res = self._get(url=url) - json_response = json.loads(res.text) - the_sequence = json_response["data"]["items"][0] - return Sequence.from_JSON(the_sequence) - - def get_sequence_by_external_id(self, external_id: str) -> Sequence: - """Returns a Sequence object containing the requested sequence. - - Args: - external_id (int): External ID of the sequence to look up - - Returns: - test_experimental.dto.Sequence: A data object containing the requested sequence. - """ - url = "/sequences" - params = {"externalId": external_id} - res = self._get(url=url, params=params) - json_response = json.loads(res.text) - the_sequence = json_response["data"]["items"][0] - return Sequence.from_JSON(the_sequence) - - def list_sequences(self, external_id: str = None): - """Returns a list of Sequence objects. - - Args: - external_id (int, optional): External ID of the sequence to look up - - Returns: - List[test_experimental.dto.Sequence]: A data object containing the requested sequence. - """ - url = "/sequences" - params = {"externalId": external_id} - res = self._get(url=url, params=params) - json_response = json.loads(res.text) - sequences = json_response["data"]["items"] - return [Sequence.from_JSON(seq) for seq in sequences] - - def delete_sequence_by_id(self, id: int) -> None: - """Deletes the sequence with the given id. - - Args: - id (int): ID of the sequence to delete - - Returns: - None - """ - url = "/sequences/{}".format(id) - self._delete(url=url) - - def post_data_to_sequence(self, id: int, rows: List[Row]) -> None: - """Posts data to a sequence. - - Args: - id (int): ID of the sequence. - rows (list): List of rows with the data. - - Returns: - None - """ - url = "/sequences/{}/postdata".format(id) - body = {"items": [{"rows": [row.__dict__ for row in rows]}]} - self._post(url, body=body) - - def get_data_from_sequence( - self, - id: int, - inclusive_from: int = None, - inclusive_to: int = None, - limit: int = 100, - column_ids: List[int] = None, - ) -> SequenceDataResponse: - """Gets data from the given sequence. - - Args: - id (int): id of the sequence. - inclusive_from (int): Row number to get from (inclusive). If set to None, you'll get data from the first row - that exists. - inclusive_to (int): Row number to get to (inclusive). If set to None, you'll get data to the last row that - exists (depending on the limit). - limit (int): How many rows to return. - column_ids (List[int]): ids of the columns to get data for. - - Returns: - client.test_experimental.sequences.SequenceDataResponse: A data object containing the requested sequence. - """ - url = "/sequences/{}/getdata".format(id) - sequenceDataRequest = SequenceDataRequest( - inclusive_from=inclusive_from, inclusive_to=inclusive_to, limit=limit, column_ids=column_ids or [] - ) - body = {"items": [sequenceDataRequest.__dict__]} - res = self._post(url=url, body=body) - json_response = json.loads(res.text) - the_data = json_response["data"]["items"][0] - return SequenceDataResponse.from_JSON(the_data) diff --git a/cognite/client/experimental/time_series.py b/cognite/client/experimental/time_series.py deleted file mode 100644 index 5459ddd9ff..0000000000 --- a/cognite/client/experimental/time_series.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -from copy import copy -from typing import List - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteResponse - - -class TimeSeriesResponse(CogniteResponse): - """Time series Response Object""" - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"] - - def to_pandas(self, include_metadata: bool = False): - """Returns data as a pandas dataframe - - Args: - include_metadata (bool): Whether or not to include metadata fields in the resulting dataframe - """ - items = copy(self.internal_representation["data"]["items"]) - if items and items[0].get("metadata") is None: - return pd.DataFrame(items) - for d in items: - if d.get("metadata"): - metadata = d.pop("metadata") - if include_metadata: - d.update(metadata) - return pd.DataFrame(items) - - -class TimeSeriesClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.6", **kwargs) - - def delete_time_series_by_id(self, ids: List[int]) -> None: - """Delete multiple time series by id. - - Args: - ids (List[int]): IDs of time series to delete. - - Returns: - None - - Examples: - Delete a single time series by id:: - - client = CogniteClient() - - client.time_series.delete_time_series_by_id(ids=[my_ts_id]) - """ - url = "/timeseries/delete" - body = {"items": ids} - self._post(url, body=body) - - def get_time_series_by_id(self, id: int) -> TimeSeriesResponse: - """Returns a TimeseriesResponse object containing the requested timeseries. - - Args: - id (int): ID of timeseries to look up - - Returns: - client.experimental.time_series.TimeSeriesResponse: A data object containing the requested timeseries. - """ - url = "/timeseries/{}".format(id) - params = {} - res = self._get(url=url, params=params) - return TimeSeriesResponse(res.json()) - - def get_multiple_time_series_by_id(self, ids: List[int]) -> TimeSeriesResponse: - """Returns a TimeseriesResponse object containing the requested timeseries. - - Args: - ids (List[int]): IDs of timeseries to look up - - Returns: - client.experimental.time_series.TimeSeriesResponse: A data object containing the requested timeseries with several - getter methods with different output formats. - """ - url = "/timeseries/byids" - body = {"items": ids} - params = {} - res = self._post(url=url, body=body, params=params) - return TimeSeriesResponse(res.json()) - - def search_for_time_series( - self, - name=None, - description=None, - query=None, - unit=None, - is_string=None, - is_step=None, - metadata=None, - asset_ids=None, - asset_subtrees=None, - min_created_time=None, - max_created_time=None, - min_last_updated_time=None, - max_last_updated_time=None, - **kwargs - ) -> TimeSeriesResponse: - """Returns a TimeSeriesResponse object containing the search results. - - Args: - name (str): Prefix and fuzzy search on name. - description (str): Prefix and fuzzy search on description. - query (str): Search on name and description using wildcard search on each of the words (separated by spaces). - Retrieves results where at least on word must match. Example: "some other" - unit (str): Filter on unit (case-sensitive) - is_string (bool): Filter on whether the ts is a string ts or not. - is_step (bool): Filter on whether the ts is a step ts or not. - metadata (Dict): Filter out time series that do not match these metadata fields and values (case-sensitive). - Format is {"key1": "val1", "key2", "val2"} - asset_ids (List): Filter out time series that are not linked to any of these assets. Format is [12,345,6,7890]. - asset_subtrees (List): Filter out time series that are not linked to assets in the subtree rooted at these assets. - Format is [12,345,6,7890]. - min_created_time (int): Filter out time series with createdTime before this. Format is milliseconds since epoch. - max_created_time (int): Filter out time series with createdTime after this. Format is milliseconds since epoch. - min_last_updated_time (int): Filter out time series with lastUpdatedTime before this. Format is milliseconds since epoch. - max_last_updated_time (int): Filter out time series with lastUpdatedTime after this. Format is milliseconds since epoch. - - Keyword Arguments: - sort (str): "createdTime" or "lastUpdatedTime". Field to be sorted. - If not specified, results are sorted by relevance score. - dir (str): "asc" or "desc". Only applicable if sort is specified. Default 'desc'. - limit (int): Return up to this many results. Maximum is 1000. Default is 25. - offset (int): Offset from the first result. Sum of limit and offset must not exceed 1000. Default is 0. - boost_name (bool): Whether or not boosting name field. This option is test_experimental and can be changed. - - Returns: - client.experimental.time_series.TimeSeriesResponse: A data object containing the requested timeseries with several getter methods with different - output formats. - """ - url = "/timeseries/search" - params = { - "name": name, - "description": description, - "query": query, - "unit": unit, - "isString": is_string, - "isStep": is_step, - "metadata": str(metadata) if metadata is not None else None, - "assetIds": str(asset_ids) if asset_ids is not None else None, - "assetSubtrees": str(asset_subtrees) if asset_subtrees is not None else None, - "minCreatedTime": min_created_time, - "maxCreatedTime": max_created_time, - "minLastUpdatedTime": min_last_updated_time, - "maxLastUpdatedTime": max_last_updated_time, - "sort": kwargs.get("sort"), - "dir": kwargs.get("dir"), - "limit": kwargs.get("limit", self._LIMIT), - "offset": kwargs.get("offset"), - "boostName": kwargs.get("boost_name"), - } - res = self._get(url, params=params) - return TimeSeriesResponse(res.json()) diff --git a/cognite/client/stable/__init__.py b/cognite/client/stable/__init__.py deleted file mode 100644 index 8b13789179..0000000000 --- a/cognite/client/stable/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/cognite/client/stable/assets.py b/cognite/client/stable/assets.py deleted file mode 100644 index 4c014b2072..0000000000 --- a/cognite/client/stable/assets.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from typing import Dict, List - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResource, CogniteResponse - - -class AssetResponse(CogniteResponse): - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item.get("id") - self.name = item.get("name") - self.depth = item.get("depth") - self.description = item.get("description") - self.created_time = item.get("createdTime") - self.last_updated_time = item.get("lastUpdatedTime") - self.metadata = item.get("metadata") - self.parent_id = item.get("parentId") - self.path = item.get("path") - - def to_pandas(self): - """Returns data as a pandas dataframe""" - if len(self.to_json()) > 0: - asset = self.to_json().copy() - # Hack to avoid path ending up as first element in dict as from_dict will fail - path = asset.pop("path") - df = pd.DataFrame.from_dict(asset, orient="index") - df.loc["path"] = [path] - return df - return pd.DataFrame() - - -class AssetListResponse(CogniteCollectionResponse): - """Assets Response Object""" - - _RESPONSE_CLASS = AssetResponse - - def to_pandas(self): - """Returns data as a pandas dataframe""" - if len(self.to_json()) > 0: - return pd.DataFrame(self.internal_representation["data"]["items"]) - return pd.DataFrame() - - -class Asset(CogniteResource): - """Data transfer object for assets. - - Args: - name (str): Name of asset. Often referred to as tag. - id (int): Id of asset. - parent_id (int): ID of parent asset, if any. - description (str): Description of asset. - metadata (dict): Custom , application specific metadata. String key -> String Value. - ref_id (str): Reference ID used only in post request to disambiguate references to duplicate - names. - parent_name (str): Name of parent, this parent must exist in the same POST request. - parent_ref_id (str): Reference ID of parent, to disambiguate if multiple nodes have the same name. - source (str): Source of asset. - source_id (str): Source id of asset. - """ - - def __init__( - self, - name=None, - id=None, - parent_id=None, - description=None, - metadata=None, - ref_id=None, - parent_name=None, - parent_ref_id=None, - source=None, - source_id=None, - ): - self.name = name - self.id = id - self.parent_id = parent_id - self.description = description - self.metadata = metadata - self.ref_id = ref_id - self.parent_name = parent_name - self.parent_ref_id = parent_ref_id - self.source = source - self.source_id = source_id - - -class AssetsClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def get_assets( - self, name=None, path=None, description=None, metadata=None, depth=None, fuzziness=None, **kwargs - ) -> AssetListResponse: - """Returns assets matching provided description. - - Args: - name (str): The name of the asset(s) to get. - - path (List[int]): The path of the subtree to search in. - - description (str): Search query. - - metadata (dict): The metadata values used to filter the results. - - depth (int): Get sub assets up oto this many levels below the specified path. - - fuzziness (int): The degree of fuzziness in the name matching. - - Keyword Arguments: - autopaging (bool): Whether or not to automatically page through results. If set to true, limit will be - disregarded. Defaults to False. - - limit (int): The maximum number of assets to be returned. - - cursor (str): Cursor to use for paging through results. - Returns: - stable.assets.AssetListResponse: A data object containing the requested assets with several getter methods with different - output formats. - - Examples: - You can fetch all assets with a maximum depth of three like this:: - - client = CogniteClient() - res = client.assets.get_assets(depth=3, autopaging=True) - print(res.to_pandas()) - - You can fetch all assets in a given subtree like this:: - - client = CogniteClient() - res = client.assets.get_assets(path=[1,2,3], autopaging=True) - print(res.to_pandas()) - """ - autopaging = kwargs.get("autopaging", False) - url = "/assets" - params = { - "name": name, - "description": description, - "path": str(path) if path else None, - "metadata": str(metadata) if metadata else None, - "depth": depth, - "fuzziness": fuzziness, - "cursor": kwargs.get("cursor"), - "limit": kwargs.get("limit", self._LIMIT) if not autopaging else self._LIMIT, - } - res = self._get(url, params=params, autopaging=autopaging) - return AssetListResponse(res.json()) - - def get_asset(self, asset_id) -> AssetResponse: - """Returns the asset with the provided assetId. - - Args: - asset_id (int): The asset id of the top asset to get. - - Returns: - stable.assets.AssetResponse: A data object containing the requested assets with several getter methods with different - output formats. - Examples: - You can fetch a single asset like this:: - - client = CogniteClient() - res = client.assets.get_asset(asset_id=123) - print(res) - """ - url = "/assets/{}/subtree".format(asset_id) - res = self._get(url) - return AssetResponse(res.json()) - - def get_asset_subtree(self, asset_id, depth=None, **kwargs) -> AssetListResponse: - """Returns asset subtree of asset with provided assetId. - - Args: - asset_id (int): The asset id of the top asset to get. - - depth (int): Get subassets this many levels below the top asset. - - Keyword Arguments: - limit (int): The maximum nuber of assets to be returned. - - cursor (str): Cursor to use for paging through results. - - autopaging (bool): Whether or not to automatically page through results. If set to true, limit will be - disregarded. Defaults to False. - Returns: - stable.assets.AssetListResponse: A data object containing the requested assets with several getter methods - with different output formats. - - Examples: - You can fetch an asset subtree like this:: - - client = CogniteClient() - res = client.assets.get_asset_subtree(asset_id=123, depth=2) - print(res.to_pandas()) - """ - autopaging = kwargs.get("autopaging", False) - url = "/assets/{}/subtree".format(asset_id) - params = { - "depth": depth, - "limit": kwargs.get("limit", self._LIMIT) if not autopaging else self._LIMIT, - "cursor": kwargs.get("cursor"), - } - res = self._get(url, params=params, autopaging=autopaging) - return AssetListResponse(res.json()) - - def post_assets(self, assets: List[Asset]) -> AssetListResponse: - """Insert a list of assets. - - Args: - assets (list[stable.assets.Asset]): List of asset data transfer objects. - - Returns: - stable.assets.AssetListResponse: A data object containing the posted assets with several getter methods with different - output formats. - - Examples: - Posting an asset:: - - from cognite.client.stable.assets import Asset - - client = CogniteClient() - - my_asset = Asset("myasset") - assets_to_post = [my_asset] - res = client.assets.post_assets(assets_to_post) - print(res) - """ - url = "/assets" - items = [asset.camel_case_dict() for asset in assets] - res = self._post(url, body={"items": items}) - return AssetListResponse(res.json()) - - def delete_assets(self, asset_ids: List[int]) -> None: - """Delete a list of assets. - - Args: - asset_ids (list[int]): List of IDs of assets to delete. - - Returns: - None - - Examples: - You can delete an asset like this:: - - client = CogniteClient() - res = client.assets.delete_assets([123]) - """ - url = "/assets/delete" - body = {"items": asset_ids} - self._post(url, body=body) - - @staticmethod - def _asset_to_patch_format(id, name=None, description=None, metadata=None, source=None, source_id=None): - patch_asset = {"id": id} - if name: - patch_asset["name"] = {"set": name} - if description: - patch_asset["description"] = {"set": description} - if metadata: - patch_asset["metadata"] = {"set": metadata} - if source: - patch_asset["source"] = {"set": source} - if source_id: - patch_asset["sourceId"] = {"set": source_id} - return patch_asset - - def update_asset( - self, - asset_id: int, - name: str = None, - description: str = None, - metadata: Dict = None, - source: str = None, - source_id: str = None, - ) -> AssetResponse: - """Update an asset - - Args: - asset_id (int): The id of the asset to update - name (str, optional): The new name - description (str, optional): The new description - metadata (Dict, optional): The new metadata - source (str, optional): The new source - source_id (str, optional): The new source id - - Returns: - AssetResponse: The updated asset - """ - url = "/assets/{}/update".format(asset_id) - body = self._asset_to_patch_format(asset_id, name, description, metadata, source, source_id) - res = self._post(url, body=body) - return AssetResponse(res.json()) - - def update_assets(self, assets: List[Asset]): - """Update multiple assets - - Args: - assets (List[stable.assets.Asset]): List of assets to update - - Returns: - AssetListResponse: List of updated assets - """ - url = "/assets/update" - items = [ - self._asset_to_patch_format(a.id, a.name, a.description, a.metadata, a.source, a.source_id) for a in assets - ] - res = self._post(url, body={"items": items}) - return AssetListResponse(res.json()) - - def search_for_assets( - self, - name=None, - description=None, - query=None, - metadata=None, - asset_subtrees=None, - min_created_time=None, - max_created_time=None, - min_last_updated_time=None, - max_last_updated_time=None, - **kwargs - ) -> AssetListResponse: - """Search for assets. - - Args: - name: Prefix and fuzzy search on name. - description str: Prefix and fuzzy search on description. - query (str): Search on name and description using wildcard search on each of the words - (separated by spaces). Retrieves results where at least one word must match. - Example: 'some other' - metadata (dict): Filter out assets that do not match these metadata fields and values (case-sensitive). - Format is {"key1":"value1","key2":"value2"}. - asset_subtrees (List[int]): Filter out assets that are not linked to assets in the subtree rooted at these assets. - Format is [12,345,6,7890]. - min_created_time(str): Filter out assets with createdTime before this. Format is milliseconds since epoch. - max_created_time (str): Filter out assets with createdTime after this. Format is milliseconds since epoch. - min_last_updated_time(str): Filter out assets with lastUpdatedtime before this. Format is milliseconds since epoch. - max_last_updated_time(str): Filter out assets with lastUpdatedtime after this. Format is milliseconds since epoch. - - Keyword Args: - sort (str): Field to be sorted. - dir (str): Sort direction (desc or asc) - limit (int): Return up to this many results. Max is 1000, default is 25. - offset (int): Offset from the first result. Sum of limit and offset must not exceed 1000. Default is 0. - boost_name (str): Whether or not boosting name field. This option is test_experimental and can be changed. - Returns: - stable.assets.AssetListResponse. - - Examples: - Searching for assets:: - - client = CogniteClient() - res = client.assets.search_for_assets(name="myasset") - print(res) - """ - url = "/assets/search" - params = { - "name": name, - "description": description, - "query": query, - "metadata": json.dumps(metadata), - "assetSubtrees": asset_subtrees, - "minCreatedTime": min_created_time, - "maxCreatedTime": max_created_time, - "minLastUpdatedTime": min_last_updated_time, - "maxLastUpdatedTime": max_last_updated_time, - "sort": kwargs.get("sort"), - "dir": kwargs.get("dir"), - "limit": kwargs.get("limit", self._LIMIT), - "offset": kwargs.get("offset"), - "boostName": kwargs.get("boost_name"), - } - - res = self._get(url, params=params) - return AssetListResponse(res.json()) diff --git a/cognite/client/stable/datapoints.py b/cognite/client/stable/datapoints.py deleted file mode 100644 index facdc0e6f3..0000000000 --- a/cognite/client/stable/datapoints.py +++ /dev/null @@ -1,833 +0,0 @@ -# -*- coding: utf-8 -*- -import io -import json -import time -from concurrent.futures import ThreadPoolExecutor as Pool -from copy import copy -from datetime import datetime -from functools import partial -from typing import List -from urllib.parse import quote - -import pandas as pd - -from cognite.client import _utils -from cognite.client._api_client import APIClient, CogniteResource, CogniteResponse -from cognite.client._auxiliary._protobuf_descriptors import _api_timeseries_data_v2_pb2 - - -class DatapointsResponse(CogniteResponse): - """Datapoints Response Object.""" - - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.name = item.get("name") - self.datapoints = item.get("datapoints") - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"][0] - - def to_pandas(self): - """Returns data as a pandas dataframe""" - return pd.DataFrame(self.internal_representation["data"]["items"][0]["datapoints"]) - - -class DatapointsQuery(CogniteResource): - """Data Query Object for Datapoints. - - Args: - name (str): Unique name of the time series. - aggregates (list): The aggregate functions to be returned. Use default if null. An empty string must - be sent to get raw data if the default is a set of aggregate functions. - granularity (str): The granularity size and granularity of the aggregates. - start (str, int, datetime): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - Example: '2d-ago' will get everything that is up to 2 days old. Can also send time in - ms since epoch or as a datetime object. - end (str, int, datetime): Get datapoints up to this time. The format is the same as for start. - """ - - def __init__(self, name, aggregates=None, granularity=None, start=None, end=None, limit=None): - self.name = name - self.aggregates = ",".join(aggregates) if aggregates is not None else None - self.granularity = granularity - self.start, self.end = _utils.interval_to_ms(start, end) - if not start: - self.start = None - if not end: - self.end = None - self.limit = limit - - def __str__(self): - return json.dumps(self.__dict__, indent=4, sort_keys=True) - - -class DatapointsResponseIterator: - """Iterator for Datapoints Response Objects.""" - - def __init__(self, datapoints_objects): - self.datapoints_objects = datapoints_objects - self.counter = 0 - - def __getitem__(self, index): - return self.datapoints_objects[index] - - def __len__(self): - return len(self.datapoints_objects) - - def __iter__(self): - return self - - def __next__(self): - if self.counter > len(self.datapoints_objects) - 1: - raise StopIteration - else: - self.counter += 1 - return self.datapoints_objects[self.counter - 1] - - -class Datapoint(CogniteResource): - """Data transfer object for datapoints. - - Args: - timestamp (Union[int, float, datetime]): The data timestamp in milliseconds since the epoch (Jan 1, 1970) or as - a datetime object. - value (Union[string, int, float]): The data value, Can be string or numeric depending on the metric. - """ - - def __init__(self, timestamp, value): - if isinstance(timestamp, datetime): - self.timestamp = _utils.datetime_to_ms(timestamp) - else: - self.timestamp = timestamp - self.value = value - - -class TimeseriesWithDatapoints(CogniteResource): - """Data transfer object for a timeseries with datapoints. - - Args: - name (str): Unique ID of time series. - datapoints (List[stable.datapoints.Datapoint]): List of datapoints in the timeseries. - """ - - def __init__(self, name, datapoints): - self.name = name - self.datapoints = datapoints - - -class LatestDatapointResponse(CogniteResponse): - """Latest Datapoint Response Object.""" - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"][0] - - def to_pandas(self): - """Returns data as a pandas dataframe""" - return pd.DataFrame([self.internal_representation["data"]["items"][0]]) - - -class DatapointsClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - self._LIMIT_AGG = 10000 - self._LIMIT = 100000 - - def get_datapoints(self, name, start, end=None, aggregates=None, granularity=None, **kwargs) -> DatapointsResponse: - """Returns a DatapointsObject containing a list of datapoints for the given query. - - This method will automate paging for the user and return all data for the given time period. - - Args: - name (str): The name of the timeseries to retrieve data for. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - Keyword Arguments: - workers (int): Number of download workers to run in parallell. Defaults to 10. - - include_outside_points (bool): No description - - protobuf (bool): Download the data using the binary protobuf format. Only applicable when getting raw data. - Defaults to True. - - limit (str): Max number of datapoints to return. If limit is specified, this method will not automate - paging and will return a maximum of 100,000 dps. - - Returns: - stable.datapoints.DatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - - Examples: - Getting the last 3 days of raw datapoints for a given time series:: - - client = CogniteClient() - res = client.datapoints.get_datapoints(name="my_ts", start="3d-ago") - print(res.to_pandas()) - """ - start, end = _utils.interval_to_ms(start, end) - - if aggregates: - aggregates = ",".join(aggregates) - - if kwargs.get("limit"): - return self._get_datapoints_user_defined_limit( - name, - aggregates, - granularity, - start, - end, - limit=kwargs.get("limit"), - protobuf=kwargs.get("protobuf"), - include_outside_points=kwargs.get("include_outside_points", False), - ) - - num_of_workers = kwargs.get("workers", self._num_of_workers) - if kwargs.get("include_outside_points") is True: - num_of_workers = 1 - - windows = _utils.get_datapoints_windows(start, end, granularity, num_of_workers) - - partial_get_dps = partial( - self._get_datapoints_helper_wrapper, - name=name, - aggregates=aggregates, - granularity=granularity, - protobuf=kwargs.get("protobuf", True), - include_outside_points=kwargs.get("include_outside_points", False), - ) - - with Pool(len(windows)) as p: - datapoints = p.map(partial_get_dps, windows) - - concat_dps = [] - [concat_dps.extend(el) for el in datapoints] - - return DatapointsResponse({"data": {"items": [{"name": name, "datapoints": concat_dps}]}}) - - def _get_datapoints_helper_wrapper(self, args, name, aggregates, granularity, protobuf, include_outside_points): - return self._get_datapoints_helper( - name, - aggregates, - granularity, - args["start"], - args["end"], - protobuf=protobuf, - include_outside_points=include_outside_points, - ) - - def _get_datapoints_helper(self, name, aggregates=None, granularity=None, start=None, end=None, **kwargs): - """Returns a list of datapoints for the given query. - - This method will automate paging for the given time period. - - Args: - name (str): The name of the timeseries to retrieve data for. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - Keyword Arguments: - include_outside_points (bool): No description. - - protobuf (bool): Download the data using the binary protobuf format. Only applicable when getting raw data. - Defaults to True. - - Returns: - list of datapoints: A list containing datapoint dicts. - """ - url = "/timeseries/data/{}".format(quote(name, safe="")) - - use_protobuf = kwargs.get("protobuf", True) and aggregates is None - limit = self._LIMIT if aggregates is None else self._LIMIT_AGG - - params = { - "aggregates": aggregates, - "granularity": granularity, - "limit": limit, - "start": start, - "end": end, - "includeOutsidePoints": kwargs.get("include_outside_points", False), - } - - headers = {"accept": "application/protobuf"} if use_protobuf else {} - datapoints = [] - while (not datapoints or len(datapoints[-1]) == limit) and params["end"] > params["start"]: - res = self._get(url, params=params, headers=headers) - if use_protobuf: - ts_data = _api_timeseries_data_v2_pb2.TimeseriesData() - ts_data.ParseFromString(res.content) - res = [{"timestamp": p.timestamp, "value": p.value} for p in ts_data.numericData.points] - else: - res = res.json()["data"]["items"][0]["datapoints"] - - if not res: - break - - datapoints.append(res) - latest_timestamp = int(datapoints[-1][-1]["timestamp"]) - params["start"] = latest_timestamp + (_utils.granularity_to_ms(granularity) if granularity else 1) - dps = [] - [dps.extend(el) for el in datapoints] - return dps - - def _get_datapoints_user_defined_limit(self, name, aggregates, granularity, start, end, limit, **kwargs): - """Returns a DatapointsResponse object with the requested data. - - No paging or parallelizing is done. - - Args: - name (str): The name of the timeseries to retrieve data for. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - limit (str): Max number of datapoints to return. Max is 100,000. - - Keyword Arguments: - include_outside_points (bool): No description. - - protobuf (bool): Download the data using the binary protobuf format. Only applicable when getting raw data. - Defaults to True. - Returns: - stable.datapoints.DatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/timeseries/data/{}".format(quote(name, safe="")) - - use_protobuf = kwargs.get("protobuf", True) and aggregates is None - - params = { - "aggregates": aggregates, - "granularity": granularity, - "limit": limit, - "start": start, - "end": end, - "includeOutsidePoints": kwargs.get("include_outside_points", False), - } - headers = {"accept": "application/protobuf"} if use_protobuf else {} - res = self._get(url, params=params, headers=headers) - if use_protobuf: - ts_data = _api_timeseries_data_v2_pb2.TimeseriesData() - ts_data.ParseFromString(res.content) - res = [{"timestamp": p.timestamp, "value": p.value} for p in ts_data.numericData.points] - else: - res = res.json()["data"]["items"][0]["datapoints"] - - return DatapointsResponse({"data": {"items": [{"name": name, "datapoints": res}]}}) - - def _split_TimeseriesWithDatapoints_if_over_limit( - self, timeseries_with_datapoints: TimeseriesWithDatapoints, limit: int - ) -> List[TimeseriesWithDatapoints]: - """Takes a TimeseriesWithDatapoints and splits it into multiple so that each has a max number of datapoints equal - to the limit given. - - Args: - timeseries_with_datapoints (stable.datapoints.TimeseriesWithDatapoints): The timeseries with data to potentially split up. - - Returns: - A list of stable.datapoints.TimeSeriesWithDatapoints where each has a maximum number of datapoints equal to the limit given. - """ - timeseries_with_datapoints_list = [] - if len(timeseries_with_datapoints.datapoints) > limit: - i = 0 - while i < len(timeseries_with_datapoints.datapoints): - timeseries_with_datapoints_list.append( - TimeseriesWithDatapoints( - name=timeseries_with_datapoints.name, - datapoints=timeseries_with_datapoints.datapoints[i : i + limit], - ) - ) - i += limit - else: - timeseries_with_datapoints_list.append(timeseries_with_datapoints) - - return timeseries_with_datapoints_list - - def post_multi_time_series_datapoints(self, timeseries_with_datapoints: List[TimeseriesWithDatapoints]) -> None: - """Insert data into multiple timeseries. - - Args: - timeseries_with_datapoints (List[stable.datapoints.TimeseriesWithDatapoints]): The timeseries with data to insert. - - Returns: - None - - Examples: - Posting some dummy datapoints to multiple time series. This example assumes that the time series have - already been created:: - - from cognite.client.stable.datapoints import TimeseriesWithDatapoints, Datapoint - - start = 1514761200000 - my_dummy_data_1 = [Datapoint(timestamp=ms, value=i) for i, ms in range(start, start+100)] - ts_with_datapoints_1 = TimeSeriesWithDatapoints(name="ts1", datapoints=my_dummy_data_1) - - start = 1503331800000 - my_dummy_data_2 = [Datapoint(timestamp=ms, value=i) for i, ms in range(start, start+100)] - ts_with_datapoints_2 = TimeSeriesWithDatapoints(name="ts2", datapoints=my_dummy_data_2) - - my_dummy_data = [ts_with_datapoints_1, ts_with_datapoints_2] - - client = CogniteClient() - res = client.datapoints.post_multi_time_series_datapoints(my_dummy_data) - """ - url = "/timeseries/data" - - ul_dps_limit = 100000 - - # Make sure we only work with TimeseriesWithDatapoints objects that has a max number of datapoints - timeseries_with_datapoints_limited = [] - for entry in timeseries_with_datapoints: - timeseries_with_datapoints_limited.extend( - self._split_TimeseriesWithDatapoints_if_over_limit(entry, ul_dps_limit) - ) - - # Group these TimeseriesWithDatapoints if possible so that we upload as much as possible in each call to the API - timeseries_to_upload_binned = _utils.first_fit( - list_items=timeseries_with_datapoints_limited, max_size=ul_dps_limit, get_count=lambda x: len(x.datapoints) - ) - - for bin in timeseries_to_upload_binned: - body = { - "items": [ - {"name": ts_with_data.name, "datapoints": [dp.__dict__ for dp in ts_with_data.datapoints]} - for ts_with_data in bin - ] - } - self._post(url, body=body) - - def post_datapoints(self, name, datapoints: List[Datapoint]) -> None: - """Insert a list of datapoints. - - Args: - name (str): Name of timeseries to insert to. - - datapoints (List[stable.datapoints.Datapoint]): List of datapoint data transfer objects to insert. - - Returns: - None - - Examples: - Posting some dummy datapoints:: - - from cognite.client.stable.datapoints import Datapoint - - client = CogniteClient() - - start = 1514761200000 - my_dummy_data = [Datapoint(timestamp=start+off, value=off) for off in range(100)] - client.datapoints.post_datapoints(ts_name, my_dummy_data) - """ - url = "/timeseries/data/{}".format(quote(name, safe="")) - - ul_dps_limit = 100000 - i = 0 - while i < len(datapoints): - body = {"items": [dp.__dict__ for dp in datapoints[i : i + ul_dps_limit]]} - self._post(url, body=body) - i += ul_dps_limit - - def get_latest(self, name, before=None) -> LatestDatapointResponse: - """Returns a LatestDatapointObject containing the latest datapoint for the given timeseries. - - Args: - name (str): The name of the timeseries to retrieve data for. - - Returns: - stable.datapoints.LatestDatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - - Examples: - Get the latest datapoint from a time series before time x:: - - client = CogniteClient() - x = 1514761200000 - client.datapoints.get_latest(name="my_ts", before=x) - - """ - url = "/timeseries/latest/{}".format(quote(name, safe="")) - params = {"before": before} - res = self._get(url, params=params) - return LatestDatapointResponse(res.json()) - - def get_multi_time_series_datapoints( - self, datapoints_queries: List[DatapointsQuery], start, end=None, aggregates=None, granularity=None, **kwargs - ) -> DatapointsResponseIterator: - """Returns a list of DatapointsObjects each of which contains a list of datapoints for the given timeseries. - - This method will automate paging for the user and return all data for the given time period(s). - - Args: - datapoints_queries (list[stable.datapoints.DatapointsQuery]): The list of DatapointsQuery objects specifying which - timeseries to retrieve data for. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - aggregates (list, optional): The list of aggregate functions you wish to apply to the data. Valid aggregate - functions are: 'average/avg, max, min, count, sum, interpolation/int, - stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, - minute/m, second/s', or a multiple of these indicated by a number as a prefix - e.g. '12hour'. - - Keyword Arguments: - include_outside_points (bool): No description. - - Returns: - stable.datapoints.DatapointsResponseIterator: An iterator which iterates over stable.datapoints.DatapointsResponse objects. - """ - url = "/timeseries/dataquery" - start, end = _utils.interval_to_ms(start, end) - - datapoints_queries = [copy(dpq) for dpq in datapoints_queries] - num_of_dpqs_with_agg = 0 - num_of_dpqs_raw = 0 - for dpq in datapoints_queries: - if (dpq.aggregates is None and aggregates is None) or dpq.aggregates == "": - num_of_dpqs_raw += 1 - else: - num_of_dpqs_with_agg += 1 - - items = [] - for dpq in datapoints_queries: - if dpq.aggregates is None and aggregates is None: - dpq.limit = int(self._LIMIT / num_of_dpqs_raw) - else: - dpq.limit = int(self._LIMIT_AGG / num_of_dpqs_with_agg) - items.append(dpq.__dict__) - body = { - "items": items, - "aggregates": ",".join(aggregates) if aggregates is not None else None, - "granularity": granularity, - "start": start, - "includeOutsidePoints": kwargs.get("include_outside_points", False), - "end": end, - } - datapoints_responses = [] - has_incomplete_requests = True - while has_incomplete_requests: - res = self._post(url=url, body=body).json()["data"]["items"] - datapoints_responses.append(res) - has_incomplete_requests = False - for i, dpr in enumerate(res): - dpq = datapoints_queries[i] - if len(dpr["datapoints"]) == dpq.limit: - has_incomplete_requests = True - latest_timestamp = dpr["datapoints"][-1]["timestamp"] - ts_granularity = granularity if dpq.granularity is None else dpq.granularity - next_start = latest_timestamp + (_utils.granularity_to_ms(ts_granularity) if ts_granularity else 1) - else: - next_start = end - 1 - if datapoints_queries[i].end: - next_start = datapoints_queries[i].end - 1 - datapoints_queries[i].start = next_start - - results = [{"data": {"items": [{"name": dpq.name, "datapoints": []}]}} for dpq in datapoints_queries] - for res in datapoints_responses: - for i, ts in enumerate(res): - results[i]["data"]["items"][0]["datapoints"].extend(ts["datapoints"]) - return DatapointsResponseIterator([DatapointsResponse(result) for result in results]) - - def get_datapoints_frame(self, time_series, aggregates, granularity, start, end=None, **kwargs) -> pd.DataFrame: - """Returns a pandas dataframe of datapoints for the given timeseries all on the same timestamps. - - This method will automate paging for the user and return all data for the given time period. - - Args: - time_series (list): The list of timeseries names to retrieve data for. Each timeseries can be either a string - containing the timeseries or a dictionary containing the names of thetimeseries and a - list of specific aggregate functions. - - aggregates (list): The list of aggregate functions you wish to apply to the data for which you have not - specified an aggregate function. Valid aggregate functions are: 'average/avg, max, min, - count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - Keyword Arguments: - limit (str): Max number of rows to return. If limit is specified, this method will not automate - paging and will return a maximum of 100,000 rows. - - workers (int): Number of download workers to run in parallell. Defaults to 10. - - Returns: - pandas.DataFrame: A pandas dataframe containing the datapoints for the given timeseries. The datapoints for all the - timeseries will all be on the same timestamps. - - Examples: - Get a dataframe of aggregated time series data:: - - client = CogniteClient() - - res = client.datapoints.get_datapoints_frame(time_series=["ts1", "ts2"], - aggregates=["avg"], granularity="30s", start="1w-ago") - - print(res) - - The ``timeseries`` parameter can take a list of strings and/or dicts on the following formats. - This is useful for specifying aggregate functions on a per time series level:: - - Using strings: - ['', ''] - - Using dicts: - [{'name': '', 'aggregates': ['', '']}, - {'name': '', 'aggregates': []}] - - Using both: - ['', {'name': '', 'aggregates': ['', '']}] - """ - if not isinstance(time_series, list): - raise ValueError("time_series should be a list") - start, end = _utils.interval_to_ms(start, end) - - if kwargs.get("limit"): - return self._get_datapoints_frame_user_defined_limit( - time_series, aggregates, granularity, start, end, limit=kwargs.get("limit") - ) - - num_of_workers = kwargs.get("workers") or self._num_of_workers - - windows = _utils.get_datapoints_windows(start, end, granularity, num_of_workers) - - partial_get_dpsf = partial( - self._get_datapoints_frame_helper_wrapper, - time_series=time_series, - aggregates=aggregates, - granularity=granularity, - ) - - with Pool(len(windows)) as p: - dataframes = p.map(partial_get_dpsf, windows) - - df = pd.concat(dataframes).drop_duplicates(subset="timestamp").reset_index(drop=True) - - return df - - def _get_datapoints_frame_helper_wrapper(self, args, time_series, aggregates, granularity): - return self._get_datapoints_frame_helper(time_series, aggregates, granularity, args["start"], args["end"]) - - def _get_datapoints_frame_helper(self, time_series, aggregates, granularity, start=None, end=None): - """Returns a pandas dataframe of datapoints for the given timeseries all on the same timestamps. - - This method will automate paging for the user and return all data for the given time period. - - Args: - time_series (list): The list of timeseries names to retrieve data for. Each timeseries can be either a string containing the - ts name or a dictionary containing the ts name and a list of specific aggregate functions. - - aggregates (list): The list of aggregate functions you wish to apply to the data for which you have not - specified an aggregate function. Valid aggregate functions are: 'average/avg, max, min, - count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - Returns: - pandas.DataFrame: A pandas dataframe containing the datapoints for the given timeseries. The datapoints for all the - timeseries will all be on the same timestamps. - - Note: - The ``timeseries`` parameter can take a list of strings and/or dicts on the following formats:: - - Using strings: - ['', ''] - - Using dicts: - [{'name': '', 'aggregates': ['', '']}, - {'name': '', 'aggregates': []}] - - Using both: - ['', {'name': '', 'aggregates': ['', '']}] - """ - url = "/timeseries/dataframe" - num_aggregates = 0 - for ts in time_series: - if isinstance(ts, str) or ts.get("aggregates") is None: - num_aggregates += len(aggregates) - else: - num_aggregates += len(ts["aggregates"]) - - per_tag_limit = int(self._LIMIT / num_aggregates) - - body = { - "items": [ - {"name": "{}".format(ts)} - if isinstance(ts, str) - else {"name": "{}".format(ts["name"]), "aggregates": ts.get("aggregates", [])} - for ts in time_series - ], - "aggregates": aggregates, - "granularity": granularity, - "start": start, - "end": end, - "limit": per_tag_limit, - } - headers = {"accept": "text/csv"} - dataframes = [] - while (not dataframes or dataframes[-1].shape[0] == per_tag_limit) and body["end"] > body["start"]: - res = self._post(url=url, body=body, headers=headers) - dataframes.append( - pd.read_csv(io.StringIO(res.content.decode(res.encoding if res.encoding else res.apparent_encoding))) - ) - if dataframes[-1].empty: - break - latest_timestamp = int(dataframes[-1].iloc[-1, 0]) - body["start"] = latest_timestamp + _utils.granularity_to_ms(granularity) - return pd.concat(dataframes).reset_index(drop=True) - - def _get_datapoints_frame_user_defined_limit(self, time_series, aggregates, granularity, start, end, limit): - """Returns a DatapointsResponse object with the requested data. - - No paging or parallelizing is done. - - Args: - time_series (List[str]): The list of timeseries names to retrieve data for. Each timeseries can be either a string containing the - ts name or a dictionary containing the ts name and a list of specific aggregate functions. - - aggregates (list): The list of aggregate functions you wish to apply to the data. Valid aggregate functions - are: 'average/avg, max, min, count, sum, interpolation/int, stepinterpolation/step'. - - granularity (str): The granularity of the aggregate values. Valid entries are : 'day/d, hour/h, minute/m, - second/s', or a multiple of these indicated by a number as a prefix e.g. '12hour'. - - start (Union[str, int, datetime]): Get datapoints after this time. Format is N[timeunit]-ago where timeunit is w,d,h,m,s. - E.g. '2d-ago' will get everything that is up to 2 days old. Can also send time in ms since - epoch or a datetime object which will be converted to ms since epoch UTC. - - end (Union[str, int, datetime]): Get datapoints up to this time. Same format as for start. - - limit (int): Max number of rows to retrieve. Max is 100,000. - - Returns: - stable.datapoints.DatapointsResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/timeseries/dataframe" - body = { - "items": [ - {"name": "{}".format(ts)} - if isinstance(ts, str) - else {"name": "{}".format(ts["name"]), "aggregates": ts.get("aggregates", [])} - for ts in time_series - ], - "aggregates": aggregates, - "granularity": granularity, - "start": start, - "end": end, - "limit": limit, - } - - headers = {"accept": "text/csv"} - res = self._post(url=url, body=body, headers=headers) - df = pd.read_csv(io.StringIO(res.content.decode(res.encoding if res.encoding else res.apparent_encoding))) - - return df - - def post_datapoints_frame(self, dataframe) -> None: - """Write a dataframe. - The dataframe must have a 'timestamp' column with timestamps in milliseconds since epoch. - The names of the remaining columns specify the names of the time series to which column contents will be written. - Said time series must already exist. - - Args: - dataframe (pandas.DataFrame): Pandas DataFrame Object containing the time series. - - Returns: - None - - Examples: - Post a dataframe with white noise:: - - client = CogniteClient() - ts_name = 'NOISE' - - start = datetime(2018, 1, 1) - # The scaling by 1000 is important: timestamp() returns seconds - x = [(start + timedelta(days=d)).timestamp() * 1000 for d in range(100)] - y = np.random.normal(0, 1, 100) - - # The time column must be called precisely 'timestamp' - df = pd.DataFrame({'timestamp': x, ts_name: y}) - - client.datapoints.post_datapoints_frame(df) - """ - - try: - timestamp = dataframe.timestamp - names = dataframe.drop(["timestamp"], axis=1).columns - except: - raise ValueError("DataFrame not on a correct format") - - for name in names: - data_points = [Datapoint(int(timestamp.iloc[i]), dataframe[name].iloc[i]) for i in range(0, len(dataframe))] - self.post_datapoints(name, data_points) - - def live_data_generator(self, name, update_frequency=1): - """Generator function which continously polls latest datapoint of a timeseries and yields new datapoints. - - Args: - name (str): Name of timeseries to get latest datapoints for. - - update_frequency (float): Frequency to pull for data in seconds. - - Yields: - dict: Dictionary containing timestamp and value of latest datapoint. - """ - last_timestamp = self.get_latest(name).to_json()["timestamp"] - while True: - latest = self.get_latest(name).to_json() - if last_timestamp == latest["timestamp"]: - time.sleep(update_frequency) - else: - yield latest - last_timestamp = latest["timestamp"] diff --git a/cognite/client/stable/events.py b/cognite/client/stable/events.py deleted file mode 100644 index 236daeec98..0000000000 --- a/cognite/client/stable/events.py +++ /dev/null @@ -1,273 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from copy import deepcopy -from typing import List - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResource, CogniteResponse - - -class EventResponse(CogniteResponse): - """Event Response Object.""" - - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item.get("id") - self.type = item.get("type") - self.sub_type = item.get("subType") - self.description = item.get("description") - self.asset_ids = item.get("assetIds") - self.created_time = item.get("createdTime") - self.start_time = item.get("startTime") - self.end_time = item.get("endTime") - self.last_updated_time = item.get("lastUpdatedTime") - self.metadata = item.get("metadata") - - def to_pandas(self): - event = self.to_json().copy() - if event.get("metadata"): - event.update(event.pop("metadata")) - - # Hack to avoid assetIds ending up as first element in dict as from_dict will fail - asset_ids = event.pop("assetIds") - df = pd.DataFrame.from_dict(event, orient="index") - df.loc["assetIds"] = [asset_ids] - return df - - -class EventListResponse(CogniteCollectionResponse): - """Event List Response Object.""" - - _RESPONSE_CLASS = EventResponse - - def __init__(self, internal_representation): - super().__init__(internal_representation) - - def to_pandas(self): - items = deepcopy(self.to_json()) - for d in items: - if d.get("metadata"): - d.update(d.pop("metadata")) - return pd.DataFrame(items) - - -class Event(CogniteResource): - """Data transfer object for events. - - Args: - start_time (int): Start time of the event in ms since epoch. - end_time (int): End time of the event in ms since epoch. - description (str): Textual description of the event. - type (str): Type of the event, e.g. 'failure'. - subtype (str): Subtype of the event, e.g. 'electrical'. - metadata (dict): Customizable extra data about the event. - asset_ids (list[int]): List of Asset IDs of related equipments that this event relates to. - """ - - def __init__( - self, start_time=None, end_time=None, description=None, type=None, subtype=None, metadata=None, asset_ids=None - ): - self.start_time = start_time - self.end_time = end_time - self.description = description - self.type = type - self.subtype = subtype - self.metadata = metadata - self.asset_ids = asset_ids - - -class EventsClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def get_event(self, event_id: int) -> EventResponse: - """Returns a EventResponse containing an event matching the id. - - Args: - event_id (int): The event id. - - Returns: - stable.events.EventResponse: A data object containing the requested event. - - Examples: - Getting an event:: - - client = CogniteClient() - res = client.events.get_event(123) - print(res) - """ - url = "/events/{}".format(event_id) - res = self._get(url) - return EventResponse(res.json()) - - def get_events(self, type=None, sub_type=None, asset_id=None, **kwargs) -> EventListResponse: - """Returns an EventListReponse object containing events matching the query. - - Args: - type (str): Type (class) of event, e.g. 'failure'. - sub_type (str): Sub-type of event, e.g. 'electrical'. - asset_id (int): Return events associated with this assetId. - Keyword Arguments: - sort (str): Sort descending or ascending. Default 'ASC'. - cursor (str): Cursor to use for paging through results. - limit (int): Return up to this many results. Maximum is 10000. Default is 25. - has_description (bool): Return only events that have a textual description. Default null. False gives only - those without description. - min_start_time (string): Only return events from after this time. - max_start_time (string): Only return events form before this time. - autopaging (bool): Whether or not to automatically page through results. If set to true, limit will be - disregarded. Defaults to False. - - Returns: - stable.events.EventListResponse: A data object containing the requested event. - - Examples: - Getting all events of a given type:: - - client = CogniteClient() - res = client.events.get_events(type="a special type", autopaging=True) - print(res.to_pandas()) - """ - autopaging = kwargs.get("autopaging", False) - url = "/events" - - params = { - "type": type, - "subtype": sub_type, - "assetId": asset_id, - "sort": kwargs.get("sort"), - "cursor": kwargs.get("cursor"), - "limit": kwargs.get("limit", 25) if not autopaging else self._LIMIT, - "hasDescription": kwargs.get("has_description"), - "minStartTime": kwargs.get("min_start_time"), - "maxStartTime": kwargs.get("max_start_time"), - } - - res = self._get(url, params=params, autopaging=autopaging) - return EventListResponse(res.json()) - - def post_events(self, events: List[Event]) -> EventListResponse: - """Adds a list of events and returns an EventListResponse object containing created events. - - Args: - events (List[stable.events.Event]): List of events to create. - - Returns: - stable.events.EventListResponse - - Examples: - Posting two events and linking them to an asset:: - - from cognite.client.stable.events import Event - client = CogniteClient() - - my_events = [Event(start_time=1, end_time=10, type="workorder", asset_ids=[123]), - Event(start_time=11, end_time=20, type="workorder", asset_ids=[123])] - res = client.events.post_events(my_events) - print(res) - """ - url = "/events" - items = [event.camel_case_dict() for event in events] - body = {"items": items} - res = self._post(url, body=body) - return EventListResponse(res.json()) - - def delete_events(self, event_ids: List[int]) -> None: - """Deletes a list of events. - - Args: - event_ids (List[int]): List of ids of events to delete. - - Returns: - None - - Examples: - Deleting a list of events:: - - client = CogniteClient() - res = client.events.delete_events(event_ids=[1,2,3,4,5]) - """ - url = "/events/delete" - body = {"items": event_ids} - self._post(url, body=body) - - def search_for_events( - self, - description=None, - type=None, - subtype=None, - min_start_time=None, - max_start_time=None, - min_end_time=None, - max_end_time=None, - min_created_time=None, - max_created_time=None, - min_last_updated_time=None, - max_last_updated_time=None, - metadata=None, - asset_ids=None, - asset_subtrees=None, - **kwargs - ): - """Search for events. - - Args: - description (str): Prefix and fuzzy search on description. - type (str): Filter on type (case-sensitive). - subtype (str): Filter on subtype (case-sensitive). - min_start_time (str): Filter out events with startTime before this. Format is milliseconds since epoch. - max_start_time (str): Filter out events with startTime after this. Format is milliseconds since epoch. - min_end_time (str): Filter out events with endTime before this. Format is milliseconds since epoch. - max_end_time (str): Filter out events with endTime after this. Format is milliseconds since epoch. - min_created_time(str): Filter out events with createdTime before this. Format is milliseconds since epoch. - max_created_time (str): Filter out events with createdTime after this. Format is milliseconds since epoch. - min_last_updated_time(str): Filter out events with lastUpdatedtime before this. Format is milliseconds since epoch. - max_last_updated_time(str): Filter out events with lastUpdatedtime after this. Format is milliseconds since epoch. - metadata (dict): Filter out events that do not match these metadata fields and values (case-sensitive). - Format is {"key1":"value1","key2":"value2"}. - asset_ids (List[int]): Filter out events that are not linked to any of these assets. Format is [12,345,6,7890]. - asset_subtrees (List[int]): Filter out events that are not linked to assets in the subtree rooted at these assets. - Format is [12,345,6,7890]. - - Keyword Args: - sort (str): Field to be sorted. - dir (str): Sort direction (desc or asc) - limit (int): Return up to this many results. Max is 1000, default is 25. - offset (int): Offset from the first result. Sum of limit and offset must not exceed 1000. Default is 0. - - Returns: - stable.events.EventListResponse. - - Examples: - Perform a fuzzy search on event descriptions and get the first 3 results:: - - client = CogniteClient() - res = client.events.search_for_events(description="Something like this", limit=10) - print(res) - """ - url = "/events/search" - params = { - "description": description, - "type": type, - "subtype": subtype, - "minStartTime": min_start_time, - "maxStartTime": max_start_time, - "minEndTime": min_end_time, - "maxEndTime": max_end_time, - "minCreatedTime": min_created_time, - "maxCreatedTime": max_created_time, - "minLastUpdatedTime": min_last_updated_time, - "maxLastUpdatedTime": max_last_updated_time, - "metadata": json.dumps(metadata), - "assetIds": str(asset_ids or []), - "assetSubtrees": asset_subtrees, - "sort": kwargs.get("sort"), - "dir": kwargs.get("dir"), - "limit": kwargs.get("limit", self._LIMIT), - "offset": kwargs.get("offset"), - } - - res = self._get(url, params=params) - return EventListResponse(res.json()) diff --git a/cognite/client/stable/files.py b/cognite/client/stable/files.py deleted file mode 100644 index 185f0cb66d..0000000000 --- a/cognite/client/stable/files.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import warnings -from copy import copy -from typing import Dict, List, Union - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResponse - - -class FileInfoResponse(CogniteResponse): - """File Info Response Object. - - Args: - id (int): ID given by the API to the file. - file_name (str): File name. Max length is 256. - directory (str): Directory containing the file. Max length is 512. - source (dict): Source that this file comes from. Max length is 256. - file_type (str): File type. E.g. pdf, css, spreadsheet, .. Max length is 64. - metadata (dict): Customized data about the file. - asset_ids (list[str]): Names of assets related to this file. - uploaded (bool): Whether or not the file is uploaded. - uploaded_at (int): Epoc thime (ms) when the file was uploaded succesfully. - """ - - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.internal_representation["data"]["items"][0] - self.id = item.get("id") - self.file_name = item.get("fileName") - self.directory = item.get("directory") - self.source = item.get("source") - self.file_type = item.get("fileType") - self.metadata = item.get("metadata") - self.asset_ids = item.get("assetIds") - self.uploaded = item.get("uploaded") - self.uploaded_at = item.get("uploadedAt") - - def to_pandas(self): - file_info = copy(self.to_json()) - if file_info.get("metadata"): - file_info.update(file_info.pop("metadata")) - return pd.DataFrame.from_dict(file_info, orient="index") - - -class FileListResponse(CogniteCollectionResponse): - """File List Response Object""" - - _RESPONSE_CLASS = FileInfoResponse - - def to_pandas(self): - return pd.DataFrame(self.to_json()) - - -class FilesClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def upload_file( - self, file_name, file_path=None, directory=None, source=None, file_type=None, content_type=None, **kwargs - ) -> Dict: - """Upload metadata about a file and get an upload link. - - The link will expire after 30 seconds if not resumable. A resumable upload link is default. Such a link is one-time - use and expires after one week. For more information, check this link: - https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload. Use PUT request to upload file with the - link returned. - - If file_path is specified, the file will be uploaded directly by the SDK. - - Args: - file_name (str): File name. Max length is 256. - - file_path (str, optional): Path of file to upload, if omitted a upload link will be returned. - - content_type (str, optional): MIME type of your file. Required if file_path is specified. - - directory (str, optional): Directory containing the file. Max length is 512. - - source (str, optional): Source that this file comes from. Max length is 256. - - file_type (str, optional): File type. E.g. pdf, css, spreadsheet, .. Max length is 64. - - Keyword Args: - metadata (dict): Customized data about the file. - - asset_ids (list): IDs of assets related to this file. - - resumable (bool): Whether to generate a resumable URL or not. Default is true. - - overwrite (bool): Whether to overwrite existing data if duplicate or not. Default is false. - - Returns: - Dict: A dictionary containing the field fileId and optionally also uploadURL if file_path is omitted. - - Examples: - Upload a file and link it to an asset:: - - client = CogniteClient() - res = client.files.upload_file(file_name="myfile", file_path="/path/to/my/file.txt", - content_type="text/plain", asset_ids=[123]) - file_id = res["fileId"] - """ - url = "/files/initupload" - - headers = {"X-Upload-Content-Type": content_type} - - params = {"resumable": kwargs.get("resumable", True), "overwrite": kwargs.get("overwrite", False)} - - body = { - "fileName": file_name, - "directory": directory, - "source": source, - "fileType": file_type, - "metadata": kwargs.get("metadata", None), - "assetIds": kwargs.get("asset_ids", None), - } - res_storage = self._post(url=url, body=body, headers=headers, params=params) - result = res_storage.json()["data"] - if file_path: - if not content_type: - warning = "content_type should be specified when directly uploading the file." - warnings.warn(warning) - headers = {"content-length": str(os.path.getsize(file_path))} - with open(file_path, "rb") as file: - self._request_session.put(result["uploadURL"], data=file, headers=headers) - result.pop("uploadURL") - return result - - def download_file(self, id: int, get_contents: bool = False) -> Union[str, bytes]: - """Get list of files matching query. - - Args: - id (int): Path to file to upload, if omitted a upload link will be returned. - - get_contents (bool, optional): Boolean to determince whether or not to return file contents as string. - Default is False and download url is returned. - - Returns: - Union[str, bytes]: Download link if get_contents is False else file contents. - - Examples: - Get a download url for the file:: - - client = CogniteClient() - res = client.files.download_file(id=12345) - download_url = res["downloadUrl"] - - Download a file:: - - client = CogniteClient() - file_bytes = client.files.download_file(id=12345, get_contents=True) - - """ - url = "/files/{}/downloadlink".format(id) - res = self._get(url=url) - if get_contents: - dl_link = res.json()["data"] - res = self._request_session.get(dl_link) - return res.content - return res.json()["data"] - - def delete_files(self, file_ids) -> List: - """Delete - - Args: - file_ids (list[int]): List of IDs of files to delete. - - Returns: - List of files deleted and files that failed to delete. - - Examples: - Delete two files:: - - client = CogniteClient() - res = client.files.delete_files([123, 234]) - """ - url = "/files/delete" - body = {"items": file_ids} - res = self._post(url, body=body) - return res.json()["data"] - - def list_files(self, name=None, directory=None, file_type=None, source=None, **kwargs) -> FileListResponse: - """Get list of files matching query. - - Args: - name (str, optional): List all files with this name. - - directory (str, optional): Directory to list files from. - - source (str, optional): List files coming from this source. - - file_type (str, optional): Type of files to list. - - Keyword Args: - asset_id (list): Returns all files associated with this asset id. - - sort (str): Sort descending or ascending. 'ASC' or 'DESC'. - - limit (int): Number of results to return. - - is_uploaded (bool): List only uploaded files if true. If false, list only other files. If not set, - list all files without considering whether they are uploaded or not. - - autopaging (bool): Whether or not to automatically page through results. If set to true, limit will - be disregarded. Defaults to False. - - cursor (str): Cursor to use for paging through results. - - Returns: - stable.files.FileListResponse: A data object containing the requested files information. - Examples: - List all files in a given directory:: - - client = CogniteClient() - res = client.files.list_files(directory="allfiles/myspecialfiles", autopaging=True) - print(res.to_pandas()) - """ - autopaging = kwargs.get("autopaging", False) - url = "/files" - params = { - "assetId": kwargs.get("asset_id"), - "dir": directory, - "name": name, - "type": file_type, - "source": source, - "isUploaded": kwargs.get("is_uploaded"), - "sort": kwargs.get("sort"), - "limit": kwargs.get("limit", self._LIMIT) if not autopaging else self._LIMIT, - "cursor": kwargs.get("cursor"), - } - - res = self._get(url=url, params=params, autopaging=autopaging) - return FileListResponse(res.json()) - - def get_file_info(self, id) -> FileInfoResponse: - """Returns information about a file. - - Args: - id (int): Id of the file. - - Returns: - stable.files.FileInfoResponse: A data object containing the requested file information. - - Examples: - Get info a bout a specific file:: - - client = CogniteClient() - res = client.files.get_file_info(12345) - print(res) - """ - url = "/files/{}".format(id) - res = self._get(url) - return FileInfoResponse(res.json()) diff --git a/cognite/client/stable/login.py b/cognite/client/stable/login.py deleted file mode 100644 index 22b3d5f49c..0000000000 --- a/cognite/client/stable/login.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -from copy import copy - -from cognite.client._api_client import APIClient, CogniteResponse - - -class LoginStatusResponse(CogniteResponse): - """ - Attributes: - user (str): Current user - logged_in (bool): Is user logged in - project (str): Current project - project_id (str): Current project id - """ - - def __init__(self, internal_representation): - super().__init__(internal_representation) - self.user = internal_representation["data"]["user"] - self.project = internal_representation["data"]["project"] - self.project_id = internal_representation["data"]["projectId"] - self.logged_in = internal_representation["data"]["loggedIn"] - - def to_json(self): - json_repr = copy(self.__dict__) - del json_repr["internal_representation"] - return json_repr - - -class LoginClient(APIClient): - def status(self) -> LoginStatusResponse: - """Check login status - - Returns: - client.stable.login.LoginStatusResponse - - Examples: - Check the current login status:: - - client = CogniteClient() - login_status = client.login.status() - print(login_status) - - """ - return LoginStatusResponse(self._get("/login/status").json()) diff --git a/cognite/client/stable/raw.py b/cognite/client/stable/raw.py deleted file mode 100644 index 045c9f2086..0000000000 --- a/cognite/client/stable/raw.py +++ /dev/null @@ -1,247 +0,0 @@ -# -*- coding: utf-8 -*- -import json -from typing import List - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteResponse - - -class RawResponse(CogniteResponse): - """Raw Response Object.""" - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"] - - def to_pandas(self): - """Returns data as a pandas dataframe""" - return pd.DataFrame(self.to_json()) - - -class RawRow(object): - """DTO for a row in a raw database. - - The Raw API is a simple key/value-store. Each row in a table in a raw database consists of a - unique row key and a set of columns. - - Args: - key (str): Unique key for the row. - - columns (dict): A key/value-map consisting of the values in the row. - """ - - def __init__(self, key, columns): - self.key = key - self.columns = columns - - def __repr__(self): - return json.dumps(self.repr_json()) - - def repr_json(self): - return self.__dict__ - - -class RawClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def get_databases(self, limit: int = None, cursor: str = None) -> RawResponse: - """Returns a RawObject containing a list of raw databases. - - Args: - limit (int): A limit on the amount of results to return. - - cursor (str): A cursor can be provided to navigate through pages of results. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/raw" - params = {"limit": limit, "cursor": cursor} - res = self._get(url=url, params=params, headers={"content-type": "*/*"}) - return RawResponse(res.json()) - - def create_databases(self, database_names: list) -> RawResponse: - """Creates databases in the Raw API and returns the created databases. - - Args: - database_names (list): A list of databases to create. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - - """ - url = "/raw/create" - body = {"items": [{"dbName": "{}".format(database_name)} for database_name in database_names]} - res = self._post(url=url, body=body, headers={"content-type": "*/*"}) - return RawResponse(res.json()) - - def delete_databases(self, database_names: list, recursive: bool = False) -> None: - """Deletes databases in the Raw API. - - Args: - database_names (list): A list of databases to delete. - - Returns: - None - - """ - url = "/raw/delete" - body = {"items": [{"dbName": "{}".format(database_name)} for database_name in database_names]} - params = {"recursive": recursive} - self._post(url=url, body=body, params=params, headers={"content-type": "*/*"}) - - def get_tables(self, database_name: str = None, limit: int = None, cursor: str = None) -> RawResponse: - """Returns a RawObject containing a list of tables in a raw database. - - Args: - database_name (str): The database name to retrieve tables from. - - limit (int): A limit on the amount of results to return. - - cursor (str): A cursor can be provided to navigate through pages of results. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/raw/{}".format(database_name) - params = dict() - if limit is not None: - params["limit"] = limit - if cursor is not None: - params["cursor"] = cursor - res = self._get(url=url, params=params, headers={"content-type": "*/*"}) - return RawResponse(res.json()) - - def create_tables(self, database_name: str = None, table_names: list = None) -> RawResponse: - """Creates tables in the given Raw API database. - - Args: - database_name (str): The database to create tables in. - - table_names (list): The table names to create. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - - """ - url = "/raw/{}/create".format(database_name) - body = {"items": [{"tableName": "{}".format(table_name)} for table_name in table_names]} - res = self._post(url=url, body=body, headers={"content-type": "*/*"}) - return RawResponse(res.json()) - - def delete_tables(self, database_name: str = None, table_names: list = None) -> None: - """Deletes databases in the Raw API. - - Args: - database_name (str): The database to create tables in. - - table_names (list): The table names to create. - - Returns: - None - - """ - url = "/raw/{}/delete".format(database_name) - body = {"items": [{"tableName": "{}".format(table_name)} for table_name in table_names]} - self._post(url=url, body=body, headers={"content-type": "*/*"}) - - def get_rows( - self, database_name: str = None, table_name: str = None, limit: int = None, cursor: str = None - ) -> RawResponse: - """Returns a RawObject containing a list of rows. - - Args: - database_name (str): The database name to retrieve rows from. - - table_name (str): The table name to retrieve rows from. - - limit (int): A limit on the amount of results to return. - - cursor (str): A cursor can be provided to navigate through pages of results. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/raw/{}/{}".format(database_name, table_name) - params = dict() - params["limit"] = limit - params["cursor"] = cursor - res = self._get(url=url, params=params, headers={"content-type": "*/*"}) - return RawResponse(res.json()) - - def create_rows( - self, database_name: str = None, table_name: str = None, rows: List[RawRow] = None, ensure_parent=False - ) -> None: - """Creates tables in the given Raw API database. - - Args: - database_name (str): The database to create rows in. - - table_name (str): The table names to create rows in. - - rows (list[stable.raw.RawRow]): The rows to create. - - ensure_parent (bool): Create database/table if it doesn't exist already - - Returns: - None - - """ - url = "/raw/{}/{}/create".format(database_name, table_name) - - params = {} - if ensure_parent: - params = {"ensureParent": "true"} - - ul_row_limit = 1000 - i = 0 - while i < len(rows): - body = { - "items": [{"key": "{}".format(row.key), "columns": row.columns} for row in rows[i : i + ul_row_limit]] - } - self._post(url=url, body=body, headers={"content-type": "*/*"}, params=params) - i += ul_row_limit - - def delete_rows(self, database_name: str = None, table_name: str = None, rows: List[RawRow] = None) -> None: - """Deletes rows in the Raw API. - - Args: - database_name (str): The database to create tables in. - - table_name (str): The table name where the rows are at. - - rows (list): The rows to delete. - - Returns: - None - - """ - url = "/raw/{}/{}/delete".format(database_name, table_name) - body = {"items": [{"key": "{}".format(row.key), "columns": row.columns} for row in rows]} - self._post(url=url, body=body, headers={"content-type": "*/*"}) - - def get_row(self, database_name: str = None, table_name: str = None, row_key: str = None) -> RawResponse: - """Returns a RawObject containing a list of rows. - - Args: - database_name (str): The database name to retrieve rows from. - - table_name (str): The table name to retrieve rows from. - - row_key (str): The key of the row to fetch. - - Returns: - stable.raw.RawResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/raw/{}/{}/{}".format(database_name, table_name, row_key) - params = dict() - res = self._get(url=url, params=params, headers={"content-type": "*/*"}) - return RawResponse(res.json()) diff --git a/cognite/client/stable/tagmatching.py b/cognite/client/stable/tagmatching.py deleted file mode 100644 index 2139594bdd..0000000000 --- a/cognite/client/stable/tagmatching.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteResponse - - -class TagMatchingResponse(CogniteResponse): - """Tag Matching Response Object. - - In addition to the standard output formats this data object also has a to_list() method which returns a list of - names of the tag matches. - """ - - def to_json(self): - """Returns data as a json object""" - return self.internal_representation["data"]["items"] - - def to_pandas(self): - """Returns data as a pandas dataframe""" - matches = [] - for tag in self.internal_representation["data"]["items"]: - for match in tag["matches"]: - matches.append( - { - "tag": tag["tagId"], - "match": match["tagId"], - "score": match["score"], - "platform": match["platform"], - } - ) - if matches: - return pd.DataFrame(matches)[["tag", "match", "platform", "score"]] - return pd.DataFrame() - - def to_list(self, first_matches_only=True): - """Returns a list representation of the matches. - - Args: - first_matches_only (bool): Boolean determining whether or not to return only the top match for each - tag. - - Returns: - list: list of matched tags. - """ - if self.to_pandas().empty: - return [] - if first_matches_only: - return self.to_pandas().sort_values(["score", "match"]).groupby(["tag"]).first()["match"].tolist() - return self.to_pandas().sort_values(["score", "match"])["match"].tolist() - - -class TagMatchingClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def tag_matching(self, tag_ids, fuzzy_threshold=0, platform=None) -> TagMatchingResponse: - """Returns a TagMatchingObject containing a list of matched tags for the given query. - - This method takes an arbitrary string as argument and performs fuzzy matching with a user defined threshold - toward tag ids in the system. - - Args: - tag_ids (list): The tag_ids to retrieve matches for. - - fuzzy_threshold (int): The threshold to use when searching for matches. A fuzzy threshold of 0 means you only - want to accept perfect matches. Must be >= 0. - - platform (str): The platform to search on. - - Returns: - stable.tagmatching.TagMatchingResponse: A data object containing the requested data with several getter methods with different - output formats. - """ - url = "/tagmatching" - body = {"tagIds": tag_ids, "metadata": {"fuzzyThreshold": fuzzy_threshold, "platform": platform}} - res = self._post(url=url, body=body) - return TagMatchingResponse(res.json()) diff --git a/cognite/client/stable/time_series.py b/cognite/client/stable/time_series.py deleted file mode 100644 index a002097efc..0000000000 --- a/cognite/client/stable/time_series.py +++ /dev/null @@ -1,219 +0,0 @@ -# -*- coding: utf-8 -*- -from copy import deepcopy -from typing import List -from urllib.parse import quote - -import pandas as pd - -from cognite.client._api_client import APIClient, CogniteCollectionResponse, CogniteResource, CogniteResponse - - -class TimeSeriesResponse(CogniteResponse): - """Time series Response Object""" - - def __init__(self, internal_representation): - super().__init__(internal_representation) - item = self.to_json() - self.id = item.get("id") - self.name = item.get("name") - self.unit = item.get("unit") - self.is_step = item.get("isStep") - self.is_string = item.get("isString") - self.created_time = item.get("createdTime") - self.last_updated_time = item.get("lastUpdatedTime") - self.metadata = item.get("metadata") - self.asset_id = item.get("assetId") - self.description = item.get("description") - - def to_pandas(self): - """Returns data as a pandas dataframe""" - if len(self.to_json()) > 0: - ts = self.to_json().copy() - if "metadata" in ts: - # Hack to avoid path ending up as first element in dict as from_dict will fail - metadata = ts.pop("metadata") - df = pd.DataFrame.from_dict(ts, orient="index") - df.loc["metadata"] = [metadata] - else: - df = pd.DataFrame.from_dict(ts, orient="index") - return df - return pd.DataFrame() - - -class TimeSeriesListResponse(CogniteCollectionResponse): - """Time series Response Object""" - - _RESPONSE_CLASS = TimeSeriesResponse - - def to_pandas(self, include_metadata: bool = True): - """Returns data as a pandas dataframe - - Args: - include_metadata (bool): Whether or not to include metadata fields in the resulting dataframe - """ - items = deepcopy(self.internal_representation["data"]["items"]) - if items and items[0].get("metadata") is None: - return pd.DataFrame(items) - for d in items: - if d.get("metadata"): - metadata = d.pop("metadata") - if include_metadata: - d.update(metadata) - return pd.DataFrame(items) - - -class TimeSeries(CogniteResource): - """Data Transfer Object for a time series. - - Args: - name (str): Unique name of time series. - is_string (bool): Whether the time series is string valued or not. - metadata (dict): Metadata. - unit (str): Physical unit of the time series. - asset_id (int): Asset that this time series belongs to. - description (str): Description of the time series. - security_categories (list(int)): Security categories required in order to access this time series. - is_step (bool): Whether or not the time series is a step series. - - """ - - def __init__( - self, - name, - is_string=False, - metadata=None, - unit=None, - asset_id=None, - description=None, - security_categories=None, - is_step=None, - ): - self.name = name - self.is_string = is_string - self.metadata = metadata - self.unit = unit - self.asset_id = asset_id - self.description = description - self.security_categories = security_categories - self.is_step = is_step - - -class TimeSeriesClient(APIClient): - def __init__(self, **kwargs): - super().__init__(version="0.5", **kwargs) - - def get_time_series( - self, prefix=None, description=None, include_metadata=False, asset_id=None, path=None, **kwargs - ) -> TimeSeriesListResponse: - """Returns an object containing the requested timeseries. - - Args: - prefix (str): List timeseries with this prefix in the name. - - description (str): Filter timeseries taht contains this string in its description. - - include_metadata (bool): Decide if the metadata field should be returned or not. Defaults to False. - - asset_id (int): Get timeseries related to this asset. - - path (List[int]): Get timeseries under this asset path branch. - - Keyword Arguments: - limit (int): Number of results to return. - - autopaging (bool): Whether or not to automatically page through results. If set to true, limit will be - disregarded. Defaults to False. - - Returns: - stable.time_series.TimeSeriesListResponse: A data object containing the requested timeseries with several getter methods with different - output formats. - - Examples: - Get all time series for a given asset:: - - client = CogniteClient() - res = client.time_series.get_time_series(asset_id=123, autopaging=True) - print(res.to_pandas()) - """ - autopaging = kwargs.get("autopaging", False) - url = "/timeseries" - params = { - "q": prefix, - "description": description, - "includeMetadata": include_metadata, - "assetId": asset_id, - "path": str(path) if path else None, - "limit": kwargs.get("limit", self._LIMIT) if not autopaging else self._LIMIT, - } - - res = self._get(url=url, params=params, autopaging=autopaging) - return TimeSeriesListResponse(res.json()) - - def post_time_series(self, time_series: List[TimeSeries]) -> None: - """Create a new time series. - - Args: - time_series (list[stable.time_series.TimeSeries]): List of time series data transfer objects to create. - - Returns: - None - - Examples: - Create a new time series:: - - from cognite.client.stable.time_series import TimeSeries - client = CogniteClient() - - my_time_series = [TimeSeries(name="my_ts_1")] - - client.time_series.post_time_series(my_time_series) - """ - url = "/timeseries" - items = [ts.camel_case_dict() for ts in time_series] - body = {"items": items} - self._post(url, body=body) - - def update_time_series(self, time_series: List[TimeSeries]) -> None: - """Update an existing time series. - - For each field that can be updated, a null value indicates that nothing should be done. - - Args: - time_series (list[stable.time_series.TimeSeries]): List of time series data transfer objects to update. - - Returns: - None - - Examples: - Update the unit of a time series:: - - from cognite.client.stable.time_series import TimeSeries - client = CogniteClient() - - my_time_series = [TimeSeries(name="my_ts_1", unit="celsius")] - - client.time_series.update_time_series(my_time_series) - """ - url = "/timeseries" - items = [ts.camel_case_dict() for ts in time_series] - body = {"items": items} - self._put(url, body=body) - - def delete_time_series(self, name) -> None: - """Delete a timeseries. - - Args: - name (str): Name of timeseries to delete. - - Returns: - None - - Examples: - Delete a time series by name:: - - client = CogniteClient() - - client.time_series.delete_time_series(name="my_ts_1") - """ - url = "/timeseries/{}".format(quote(name, safe="")) - self._delete(url) diff --git a/cognite/client/utils/__init__.py b/cognite/client/utils/__init__.py new file mode 100644 index 0000000000..6d7cf0a06c --- /dev/null +++ b/cognite/client/utils/__init__.py @@ -0,0 +1 @@ +from cognite.client.utils._utils import ms_to_datetime, timestamp_to_ms diff --git a/cognite/client/utils/_utils.py b/cognite/client/utils/_utils.py new file mode 100644 index 0000000000..60fae2eb89 --- /dev/null +++ b/cognite/client/utils/_utils.py @@ -0,0 +1,443 @@ +"""Utilites for Cognite API SDK + +This module provides helper methods and different utilities for the Cognite API Python SDK. + +This module is protected and should not used by end-users. +""" +import functools +import heapq +import importlib +import logging +import numbers +import platform +import random +import re +import string +import time +from concurrent.futures.thread import ThreadPoolExecutor +from datetime import datetime +from decimal import Decimal +from typing import Any, Callable, Dict, List, Tuple, Union +from urllib.parse import quote + +import cognite.client +from cognite.client.exceptions import CogniteAPIError, CogniteImportError + +_unit_in_ms_without_week = {"s": 1000, "m": 60000, "h": 3600000, "d": 86400000} +_unit_in_ms = {**_unit_in_ms_without_week, "w": 604800000} + + +def datetime_to_ms(dt): + epoch = datetime.utcfromtimestamp(0) + return int((dt - epoch).total_seconds() * 1000.0) + + +def ms_to_datetime(ms: Union[int, float]) -> datetime: + """Converts milliseconds since epoch to datetime object. + + Args: + ms (Union[int, float]): Milliseconds since epoch + + Returns: + datetime: Datetime object. + + """ + return datetime.utcfromtimestamp(ms / 1000) + + +def time_string_to_ms(pattern, string, unit_in_ms): + pattern = pattern.format("|".join(unit_in_ms)) + res = re.fullmatch(pattern, string) + if res: + magnitude = int(res.group(1)) + unit = res.group(2) + return magnitude * unit_in_ms[unit] + return None + + +def granularity_to_ms(granularity: str) -> int: + ms = time_string_to_ms(r"(\d+)({})", granularity, _unit_in_ms_without_week) + if ms is None: + raise ValueError( + "Invalid granularity format: `{}`. Must be on format (s|m|h|d). E.g. '5m', '3h' or '1d'.".format( + granularity + ) + ) + return ms + + +def granularity_unit_to_ms(granularity: str) -> int: + granularity = re.sub(r"^\d+", "1", granularity) + return granularity_to_ms(granularity) + + +def time_ago_to_ms(time_ago_string: str) -> int: + """Returns millisecond representation of time-ago string""" + if time_ago_string == "now": + return 0 + ms = time_string_to_ms(r"(\d+)({})-ago", time_ago_string, _unit_in_ms) + if ms is None: + raise ValueError( + "Invalid time-ago format: `{}`. Must be on format (s|m|h|d|w)-ago or 'now'. E.g. '3d-ago' or '1w-ago'.".format( + time_ago_string + ) + ) + return ms + + +def timestamp_to_ms(timestamp: Union[int, float, str, datetime]) -> int: + """Returns the ms representation of some timestamp given by milliseconds, time-ago format or datetime object + + Args: + timestamp (Union[int, float, str, datetime]): Convert this timestamp to ms. + + Returns: + int: Milliseconds since epoch representation of timestamp + """ + if isinstance(timestamp, int): + ms = timestamp + elif isinstance(timestamp, float): + ms = int(timestamp) + elif isinstance(timestamp, str): + ms = int(round(time.time() * 1000)) - time_ago_to_ms(timestamp) + elif isinstance(timestamp, datetime): + ms = datetime_to_ms(timestamp) + else: + raise TypeError( + "Timestamp `{}` was of type {}, but must be int, float, str or datetime,".format(timestamp, type(timestamp)) + ) + + if ms < 0: + raise ValueError( + "Timestamps can't be negative - they must represent a time after 1.1.1970, but {} was provided".format(ms) + ) + + return ms + + +@functools.lru_cache(maxsize=128) +def to_camel_case(snake_case_string: str): + components = snake_case_string.split("_") + return components[0] + "".join(x.title() for x in components[1:]) + + +@functools.lru_cache(maxsize=128) +def to_snake_case(camel_case_string: str): + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", camel_case_string) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() + + +def convert_all_keys_to_camel_case(d: Dict): + new_d = {} + for k, v in d.items(): + new_d[to_camel_case(k)] = v + return new_d + + +def json_dump_default(x): + if isinstance(x, numbers.Integral): + return int(x) + if isinstance(x, Decimal): + return float(x) + if hasattr(x, "__dict__"): + return x.__dict__ + return x + + +class TasksSummary: + def __init__(self, successful_tasks, unknown_tasks, failed_tasks, results, exceptions): + self.successful_tasks = successful_tasks + self.unknown_tasks = unknown_tasks + self.failed_tasks = failed_tasks + self.results = results + self.exceptions = exceptions + + def joined_results(self, unwrap_fn: Callable = None): + unwrap_fn = unwrap_fn or (lambda x: x) + joined_results = [] + for result in self.results: + unwrapped = unwrap_fn(result) + if isinstance(unwrapped, list): + joined_results.extend(unwrapped) + else: + joined_results.append(unwrapped) + return joined_results + + def raise_compound_exception_if_failed_tasks( + self, + task_unwrap_fn: Callable = None, + task_list_element_unwrap_fn: Callable = None, + str_format_element_fn: Callable = None, + ): + if self.exceptions: + unwrap_fn = task_unwrap_fn or (lambda x: x) + if task_list_element_unwrap_fn is not None: + el_unwrap = task_list_element_unwrap_fn + successful = [] + for t in self.successful_tasks: + successful.extend([el_unwrap(el) for el in unwrap_fn(t)]) + unknown = [] + for t in self.unknown_tasks: + unknown.extend([el_unwrap(el) for el in unwrap_fn(t)]) + failed = [] + for t in self.failed_tasks: + failed.extend([el_unwrap(el) for el in unwrap_fn(t)]) + else: + successful = [unwrap_fn(t) for t in self.successful_tasks] + unknown = [unwrap_fn(t) for t in self.unknown_tasks] + failed = [unwrap_fn(t) for t in self.failed_tasks] + if isinstance(self.exceptions[0], CogniteAPIError): + exc = self.exceptions[0] + raise CogniteAPIError( + message=exc.message, + code=exc.code, + x_request_id=exc.x_request_id, + missing=exc.missing, + duplicated=exc.duplicated, + successful=successful, + failed=failed, + unknown=unknown, + unwrap_fn=str_format_element_fn, + ) + raise self.exceptions[0] + + +def execute_tasks_concurrently(func: Callable, tasks: Union[List[Tuple], List[Dict]], max_workers: int): + assert max_workers > 0, "Number of workers should be >= 1, was {}".format(max_workers) + with ThreadPoolExecutor(max_workers) as p: + futures = [] + for task in tasks: + if isinstance(task, dict): + futures.append(p.submit(func, **task)) + elif isinstance(task, tuple): + futures.append(p.submit(func, *task)) + + successful_tasks = [] + failed_tasks = [] + unknown_result_tasks = [] + results = [] + exceptions = [] + for i, f in enumerate(futures): + try: + res = f.result() + successful_tasks.append(tasks[i]) + results.append(res) + except Exception as e: + exceptions.append(e) + if isinstance(e, CogniteAPIError): + if e.code < 500: + failed_tasks.append(tasks[i]) + else: + unknown_result_tasks.append(tasks[i]) + else: + failed_tasks.append(tasks[i]) + + return TasksSummary(successful_tasks, unknown_result_tasks, failed_tasks, results, exceptions) + + +def assert_exactly_one_of_id_or_external_id(id, external_id): + assert_type(id, "id", [numbers.Integral], allow_none=True) + assert_type(external_id, "external_id", [str], allow_none=True) + has_id = id is not None + has_external_id = external_id is not None + + assert (has_id or has_external_id) and not ( + has_id and has_external_id + ), "Exactly one of id and external id must be specified" + + if has_id: + return {"id": id} + elif has_external_id: + return {"external_id": external_id} + + +def assert_at_least_one_of_id_or_external_id(id, external_id): + assert_type(id, "id", [numbers.Integral], allow_none=True) + assert_type(external_id, "external_id", [str], allow_none=True) + has_id = id is not None + has_external_id = external_id is not None + assert has_id or has_external_id, "At least one of id and external id must be specified" + if has_id: + return {"id": id} + elif has_external_id: + return {"external_id": external_id} + + +def unwrap_identifer(identifier: Dict): + if "externalId" in identifier: + return identifier["externalId"] + if "id" in identifier: + return identifier["id"] + raise ValueError("{} does not contain 'id' or 'externalId'".format(identifier)) + + +def assert_timestamp_not_in_1970(timestamp: Union[int, float, str, datetime]): + dt = ms_to_datetime(timestamp_to_ms(timestamp)) + print(dt) + assert dt > datetime( + 1971, 1, 1 + ), "You are attempting to post data in 1970. Have you forgotten to multiply your timestamps by 1000?" + + +def assert_type(var: Any, var_name: str, types: List, allow_none=False): + if var is None: + if not allow_none: + raise TypeError("{} cannot be None".format(var_name)) + elif not isinstance(var, tuple(types)): + raise TypeError("{} must be one of types {}".format(var_name, types)) + + +def interpolate_and_url_encode(path, *args): + return path.format(*[quote(str(arg), safe="") for arg in args]) + + +def local_import(*module: str): + assert_type(module, "module", [tuple]) + if len(module) == 1: + name = module[0] + try: + return importlib.import_module(name) + except ImportError as e: + raise CogniteImportError(name.split(".")[0]) from e + + modules = [] + for name in module: + try: + modules.append(importlib.import_module(name)) + except ImportError as e: + raise CogniteImportError(name.split(".")[0]) from e + return tuple(modules) + + +def get_current_sdk_version(): + return cognite.client.__version__ + + +def get_user_agent(): + sdk_version = "CognitePythonSDK/{}".format(get_current_sdk_version()) + + python_version = "{}/{} ({};{})".format( + platform.python_implementation(), platform.python_version(), platform.python_build(), platform.python_compiler() + ) + + os_version_info = [platform.release(), platform.machine(), platform.architecture()[0]] + os_version_info = [s for s in os_version_info if s] # Ignore empty strings + os_version_info = "-".join(os_version_info) + operating_system = "{}/{}".format(platform.system(), os_version_info) + + return "{} {} {}".format(sdk_version, python_version, operating_system) + + +class DebugLogFormatter(logging.Formatter): + RESERVED_ATTRS = ( + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + ) + + def __init__(self): + fmt = "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(threadName)s. %(message)s (%(filename)s:%(lineno)s)" + datefmt = "%Y-%m-%d %H:%M:%S" + super().__init__(fmt, datefmt) + + def format(self, record): + + record.message = record.getMessage() + if self.usesTime(): + record.asctime = self.formatTime(record, self.datefmt) + s = self.formatMessage(record) + s_extra_objs = [] + for attr, value in record.__dict__.items(): + if attr not in self.RESERVED_ATTRS: + s_extra_objs.append("\n - {}: {}".format(attr, value)) + for s_extra in s_extra_objs: + s += s_extra + if record.exc_info: + # Cache the traceback text to avoid converting it multiple times + # (it's constant anyway) + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + if s[-1:] != "\n": + s = s + "\n" + s = s + record.exc_text + if record.stack_info: + if s[-1:] != "\n": + s = s + "\n" + s = s + self.formatStack(record.stack_info) + return s + + +def random_string(size=100): + return "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + + +class PriorityQueue: + def __init__(self): + self.heap = [] + self.id = 0 + + def add(self, elem, priority): + heapq.heappush(self.heap, (-priority, self.id, elem)) + self.id += 1 + + def get(self): + _, _, elem = heapq.heappop(self.heap) + return elem + + def __bool__(self): + return len(self.heap) > 0 + + +def split_into_chunks(collection: Union[List, Dict], chunk_size: int) -> List[Union[List, Dict]]: + chunks = [] + if isinstance(collection, list): + for i in range(0, len(collection), chunk_size): + chunks.append(collection[i : i + chunk_size]) + return chunks + if isinstance(collection, dict): + collection = list(collection.items()) + for i in range(0, len(collection), chunk_size): + chunks.append({k: v for k, v in collection[i : i + chunk_size]}) + return chunks + raise ValueError("Can only split list or dict") + + +def _convert_time_attributes_in_dict(item: Dict) -> Dict: + TIME_ATTRIBUTES = ["start_time", "end_time", "last_updated_time", "created_time", "timestamp"] + new_item = {} + for k, v in item.items(): + if k in TIME_ATTRIBUTES: + v = ms_to_datetime(v).strftime("%Y-%m-%d %H:%M:%S") + new_item[k] = v + return new_item + + +def convert_time_attributes_to_datetime(item: Union[Dict, List[Dict]]) -> Union[Dict, List[Dict]]: + if isinstance(item, dict): + return _convert_time_attributes_in_dict(item) + if isinstance(item, list): + new_items = [] + for el in item: + new_items.append(_convert_time_attributes_in_dict(el)) + return new_items + raise TypeError("item must be dict or list of dicts") diff --git a/cognite/client/utils/_version_checker.py b/cognite/client/utils/_version_checker.py new file mode 100644 index 0000000000..8eb775458f --- /dev/null +++ b/cognite/client/utils/_version_checker.py @@ -0,0 +1,81 @@ +import argparse +import re + +import requests + + +def check_if_version_exists(package_name: str, version: str): + versions = get_all_versions(package_name) + return version in versions + + +def get_newest_version_in_major_release(package_name: str, version: str): + major, minor, micro, pr_cycle, pr_version = _parse_version(version) + versions = get_all_versions(package_name) + for v in versions: + if _is_newer_major(v, version): + major, minor, micro, pr_cycle, pr_version = _parse_version(v) + return _format_version(major, minor, micro, pr_cycle, pr_version) + + +def get_all_versions(package_name: str): + res = requests.get("https://pypi.python.org/simple/{}/#history".format(package_name)) + versions = re.findall("cognite-sdk-(\d+\.\d+.[\dabrc]+)", res.content.decode()) + return versions + + +def _is_newer_major(version_a, version_b): + major_a, minor_a, micro_a, pr_cycle_a, pr_version_a = _parse_version(version_a) + major_b, minor_b, micro_b, pr_cycle_b, pr_version_b = _parse_version(version_b) + is_newer = False + if major_a == major_b and minor_a >= minor_b and micro_a >= micro_b: + if minor_a > minor_b: + is_newer = True + else: + if micro_a > micro_b: + is_newer = True + else: + is_newer = _is_newer_pre_release(pr_cycle_a, pr_version_a, pr_cycle_b, pr_version_b) + return is_newer + + +def _is_newer_pre_release(pr_cycle_a, pr_v_a, pr_cycle_b, pr_v_b): + cycles = ["a", "b", "rc", None] + assert pr_cycle_a in cycles, "pr_cycle_a must be one of '{}', not '{}'.".format(pr_cycle_a, cycles) + assert pr_cycle_b in cycles, "pr_cycle_a must be one of '{}', not '{}'.".format(pr_cycle_b, cycles) + is_newer = False + if cycles.index(pr_cycle_a) > cycles.index(pr_cycle_b): + is_newer = True + elif cycles.index(pr_cycle_a) == cycles.index(pr_cycle_b): + if pr_v_a is not None and pr_v_b is not None and pr_v_a > pr_v_b: + is_newer = True + return is_newer + + +def _parse_version(version: str): + pattern = "(\d+)\.(\d+)\.(\d+)(?:([abrc]+)(\d+))?" + match = re.match(pattern, version) + if not match: + raise ValueError("Could not parse version {}".format(version)) + major, minor, micro, pr_cycle, pr_version = match.groups() + return int(major), int(minor), int(micro), pr_cycle, int(pr_version) if pr_version else None + + +def _format_version(major, minor, micro, pr_cycle, pr_version): + pr_cycle = pr_cycle or "" + pr_version = pr_version or "" + return "{}.{}.{}{}{}".format(major, minor, micro, pr_cycle, pr_version) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--package", required=True) + parser.add_argument("-v", "--version", required=True) + args = parser.parse_args() + + version_exists = check_if_version_exists(args.package, args.version) + + if version_exists: + print("yes") + else: + print("no") diff --git a/core-requirements.txt b/core-requirements.txt new file mode 100644 index 0000000000..0919008626 --- /dev/null +++ b/core-requirements.txt @@ -0,0 +1,4 @@ +requests>=2.21.0,<3.0.0 +pytest-cov +pytest +responses diff --git a/docs/source/cognite.rst b/docs/source/cognite.rst index 295e0b61c4..7220529f54 100644 --- a/docs/source/cognite.rst +++ b/docs/source/cognite.rst @@ -1,256 +1,713 @@ -Overview +Quickstart +========== +Authenticate +------------ +The preferred way to authenticate against the Cognite API is by setting the :code:`COGNITE_API_KEY` environment variable. All examples in this documentation require that the variable has been set. + +.. code:: bash + + $ export COGNITE_API_KEY = + +You can also pass your API key directly to the CogniteClient. + +.. code:: python + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient(api_key="your-key") + +Instantiate a new client +------------------------ +Use this code to instantiate a client and get your login status. CDF returns an object with +attributes that describe which project and service account your API key belongs to. + +.. code:: python + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> status = c.login.status() + +Read more about the `CogniteClient`_ and the functionality it exposes below. + +Plot time series +---------------- +There are several ways of plotting a time series you have fetched from the API. The easiest is to call +:code:`.plot()` on the returned :code:`TimeSeries` or :code:`TimeSeriesList` objects. By default, this plots the raw +data points for the last 24 hours. + +.. code:: python + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> my_time_series = c.time_series.retrieve(id=[1, 2]) + >>> my_time_series.plot() + +You can also pass arguments to the :code:`.plot()` method to change the start, end, aggregates, and granularity of the +request. + +.. code:: python + + >>> my_time_series.plot(start="365d-ago", end="now", aggregates=["avg"], granularity="1d") + +The :code:`Datapoints` and :code:`DatapointsList` objects that are returned when you fetch data points, also have :code:`.plot()` +methods you can use to plot the data. + +.. code:: python + + >>> from cognite.client import CogniteClient + >>> c = CogniteClient() + >>> my_datapoints = c.datapoints.retrieve( + ... id=[1, 2], + ... start="10d-ago", + ... end="now", + ... aggregates=["max"], + ... granularity="1h" + ... ) + >>> my_datapoints.plot() + +.. NOTE:: + To use the :code:`.plot()` functionality you need to install :code:`matplotlib`. + +Create an asset hierarchy +------------------------- +To create a root asset (an asset without a parent), omit the parent ID when you post the asset to the API. +To make an asset a child of an existing asset, you must specify a parent ID. + +.. code:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import Asset + >>> c = CogniteClient() + >>> my_asset = Asset(name="my first asset", parent_id=123) + >>> c.assets.create(my_asset) + +To post an entire asset hierarchy, you can describe the relations within your asset hierarchy +using the :code:`external_id` and :code:`parent_external_id` attributes on the :code:`Asset` object. You can post +an arbitrary number of assets, and the SDK will split the request into multiple requests and create the assets +in the correct order + +This example shows how to post a three levels deep asset hierarchy consisting of three assets. + +.. code:: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes import Asset + >>> c = CogniteClient() + >>> root = Asset(name="root", external_id="1") + >>> child = Asset(name="child", external_id="2", parent_external_id="1") + >>> descendant = Asset(name="descendant", external_id="3", parent_external_id="2") + >>> c.assets.create([root, child, descendant]) + +Wrap the .create() call in a try-except to get information if posting the assets fails: + +- Which assets were posted. (The request yielded a 201.) +- Which assets may have been posted. (The request yielded 5xx.) +- Which assets were not posted. (The request yielded 4xx, or was a descendant of another asset which may or may not have been posted.) + +.. code:: + + >>> from cognite.client.exceptions import CogniteAPIError + >>> try: + ... c.create([root, child, descendant]) + >>> except CogniteAPIError as e: + ... assets_posted = e.successful + ... assets_may_have_been_posted = e.unknown + ... assets_not_posted = e.failed + +Settings ======== -Cognite Client --------------- -.. autoclass:: cognite.client.CogniteClient - :members: - :member-order: bysource +Client configuration +-------------------- +You can pass configuration arguments directly to the :code:`CogniteClient` constructor, for example to configure the base url of your requests and additional headers. For a list of all configuration arguments, see the `CogniteClient`_ class definition. -Responses ---------- -.. autoclass:: cognite.client._api_client.CogniteResponse - :members: - :undoc-members: - :show-inheritance: +Environment configuration +------------------------- +You can set default configurations with these environment variables: -.. autoclass:: cognite.client._api_client.CogniteCollectionResponse - :members: - :undoc-members: - :show-inheritance: +.. code:: bash -Exceptions ----------- -.. automodule:: cognite.client.exceptions - :members: - :undoc-members: - :show-inheritance: + $ export COGNITE_API_KEY = + $ export COGNITE_BASE_URL = http://: + $ export COGNITE_CLIENT_NAME = + $ export COGNITE_MAX_WORKERS = + $ export COGNITE_TIMEOUT = + + $ export COGNITE_MAX_RETRIES = + $ export COGNITE_MAX_CONNECTION_POOL_SIZE = + $ export COGNITE_STATUS_FORCELIST = "429,502,503" + $ export COGNITE_DISABLE_GZIP = "1" + +Concurrency and connection pooling +---------------------------------- +This library does not expose API limits to the user. If your request exceeds API limits, the SDK splits your +request into chunks and performs the sub-requests in parallel. To control how many concurrent requests you send +to the API, you can either pass the :code:`max_workers` attribute when you instantiate the :code:`CogniteClient` or set the :code:`COGNITE_MAX_WORKERS` environment variable. + +If you are working with multiple instances of :code:`CogniteClient`, all instances will share the same connection pool. +If you have several instances, you can increase the max connection pool size to reuse connections if you are performing a large amount of concurrent requests. You can increase the max connection pool size by setting the :code:`COGNITE_MAX_CONNECTION_POOL_SIZE` environment variable. + +Extensions and core library +============================ +Pandas integration +------------------ +The SDK is tightly integrated with the `pandas `_ library. +You can use the :code:`.to_pandas()` method on pretty much any object and get a pandas data frame describing the data. + +This is particularly useful when you are working with time series data and with tabular data from the Raw API. + +Matplotlib integration +---------------------- +You can use the :code:`.plot()` method on any time series or data points result that the SDK returns. The method takes keyword +arguments which are passed on to the underlying matplotlib plot function, allowing you to configure for example the +size and layout of your plots. + +You need to install the matplotlib package manually: + +.. code:: bash + + $ pip install matplotlib + +:code:`cognite-sdk` vs. :code:`cognite-sdk-core` +------------------------------------------------ +If your application doesn't require the functionality from the :code:`pandas` +or :code:`numpy` dependencies, you should install the :code:`cognite-sdk-core` library. + +The two libraries are exactly the same, except that :code:`cognite-sdk-core` does not specify :code:`pandas` +or :code:`numpy` as dependencies. This means that :code:`cognite-sdk-core` only has a subset +of the features available through the :code:`cognite-sdk` package. If you attempt to use functionality +that :code:`cognite-sdk-core` does not support, a :code:`CogniteImportError` is raised. API === -Assets ------- -Client -^^^^^^ -.. autoclass:: cognite.client.stable.assets.AssetsClient +CogniteClient +------------- +.. autoclass:: cognite.client.CogniteClient :members: + :member-order: bysource + +Authentication +-------------- +Get login status +^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.login.LoginAPI.status -Data Objects + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.assets +.. automodule:: cognite.client.data_classes.login :members: - :exclude-members: AssetsClient :undoc-members: :show-inheritance: :inherited-members: -Datapoints ----------- -Client -^^^^^^ -.. autoclass:: cognite.client.stable.datapoints.DatapointsClient - :members: +Assets +------ +Retrieve an asset by id +^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.retrieve + +Retrieve multiple assets by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.retrieve_multiple + +Retrieve an asset subtree +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.retrieve_subtree + +List assets +^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.list + +Search for assets +^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.search + +Create assets +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.create -Data Objects +Delete assets +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.delete + +Update assets +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.assets.AssetsAPI.update + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.datapoints +.. automodule:: cognite.client.data_classes.assets :members: - :exclude-members: DatapointsClient - :undoc-members: :show-inheritance: - :inherited-members: + Events ------ -Client -^^^^^^ -.. autoclass:: cognite.client.stable.events.EventsClient - :members: +Retrieve an event by id +^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.retrieve + +Retrieve multiple events by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.retrieve_multiple -Data Objects +List events +^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.list + +Search for events +^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.search + +Create events +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.create + +Delete events +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.delete + +Update events +^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.events.EventsAPI.update + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.events +.. automodule:: cognite.client.data_classes.events :members: - :exclude-members: EventsClient - :undoc-members: :show-inheritance: - :inherited-members: + Files ----- -Client -^^^^^^ -.. autoclass:: cognite.client.stable.files.FilesClient - :members: +Retrieve file metadata by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.retrieve + +Retrieve multiple files' metadata by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.retrieve_multiple + +List files metadata +^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.list + +Search for files +^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.search + +Upload a file or directory +^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.upload + +Upload a string or bytes +^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.upload_bytes -Data Objects +Download files to disk +^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.download + +Download a file as bytes +^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.download_bytes + +Delete files ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.files - :members: - :exclude-members: FilesClient - :undoc-members: - :show-inheritance: - :inherited-members: +.. automethod:: cognite.client._api.files.FilesAPI.delete -Login ------ -Client -^^^^^^ -.. autoclass:: cognite.client.stable.login.LoginClient - :members: +Update files metadata +^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.files.FilesAPI.update -Data Objects +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.login +.. automodule:: cognite.client.data_classes.files :members: - :exclude-members: LoginClient - :undoc-members: :show-inheritance: - :inherited-members: -Raw ----- -Client -^^^^^^ -.. autoclass:: cognite.client.stable.raw.RawClient - :members: -Data Objects +Time series +----------- +Retrieve a time series by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.retrieve + +Retrieve multiple time series by id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.retrieve_multiple + +List time series +^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.list + +Search for time series +^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.search + +Create time series +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.create + +Delete time series +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.delete + +Update time series +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.time_series.TimeSeriesAPI.update + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.raw +.. automodule:: cognite.client.data_classes.time_series :members: - :exclude-members: RawClient - :undoc-members: :show-inheritance: - :inherited-members: -Tagmatching + +Data points ----------- -Client -^^^^^^ -.. autoclass:: cognite.client.stable.tagmatching.TagMatchingClient - :members: +Retrieve datapoints +^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.retrieve + +Retrieve pandas dataframe +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.retrieve_dataframe + +Perform data points queries +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.query + +Retrieve latest datapoint +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.retrieve_latest -Data Objects +Insert data points +^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.insert + +Insert data points into multiple time series +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.insert_multiple + +Insert pandas dataframe +^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.insert_dataframe + +Delete a range of data points +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.delete_range + +Delete ranges of data points +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. automethod:: cognite.client._api.datapoints.DatapointsAPI.delete_ranges + + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.tagmatching +.. automodule:: cognite.client.data_classes.datapoints :members: - :exclude-members: TagMatchingClient - :undoc-members: :show-inheritance: - :inherited-members: -Time Series ------------ -Client + +Raw +--- +Databases +^^^^^^^^^ +List databases +~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawDatabasesAPI.list + +Create new databases +~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawDatabasesAPI.create + +Delete databases +~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawDatabasesAPI.delete + + +Tables ^^^^^^ -.. autoclass:: cognite.client.stable.time_series.TimeSeriesClient - :members: +List tables in a database +~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawTablesAPI.list + +Create new tables in a database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawTablesAPI.create + +Delete tables from a database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawTablesAPI.delete + + +Rows +^^^^ +Get a row from a table +~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawRowsAPI.retrieve + +List rows in a table +~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawRowsAPI.list + +Insert rows into a table +~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawRowsAPI.insert -Data Objects +Delete rows from a table +~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.raw.RawRowsAPI.delete + +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.stable.time_series +.. automodule:: cognite.client.data_classes.raw :members: - :exclude-members: TimeSeriesClient - :undoc-members: :show-inheritance: - :inherited-members: -Experimental -============ -Model Hosting -------------- + +3D +-- Models ^^^^^^ -Client -~~~~~~ -.. autoclass:: cognite.client.experimental.model_hosting.models.ModelsClient - :members: +Retrieve a model by ID +~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDModelsAPI.retrieve -Data Objects -~~~~~~~~~~~~ -.. automodule:: cognite.client.experimental.model_hosting.models - :members: - :exclude-members: ModelsClient - :undoc-members: - :show-inheritance: - :inherited-members: +List models +~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDModelsAPI.list -Source Packages -^^^^^^^^^^^^^^^ -Client -~~~~~~ -.. autoclass:: cognite.client.experimental.model_hosting.source_packages.SourcePackageClient - :members: +Create models +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDModelsAPI.create -Data Objects -~~~~~~~~~~~~ -.. automodule:: cognite.client.experimental.model_hosting.source_packages - :members: - :exclude-members: SourcePackageClient - :undoc-members: - :show-inheritance: - :inherited-members: +Update models +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDModelsAPI.update + +Delete models +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDModelsAPI.delete -Schedules + +Revisions ^^^^^^^^^ -Client -~~~~~~ -.. autoclass:: cognite.client.experimental.model_hosting.schedules.SchedulesClient - :members: +Retrieve a revision by ID +~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.retrieve + +Create a revision +~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.create + +List revisions +~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.list + +Update revisions +~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.update + +Delete revisions +~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.delete + +Update a revision thumbnail +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.update_thumbnail + +List nodes +~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.list_nodes + +List ancestor nodes +~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.list_ancestor_nodes + + +Files +^^^^^ +Retrieve a 3D file +~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDFilesAPI.retrieve + +Asset mappings +^^^^^^^^^^^^^^ +Create an asset mapping +~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDAssetMappingAPI.create -Data Objects -~~~~~~~~~~~~ -.. automodule:: cognite.client.experimental.model_hosting.schedules +List asset mappings +~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDAssetMappingAPI.list + +Delete asset mappings +~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.three_d.ThreeDAssetMappingAPI.delete + +.. + Reveal + ^^^^^^ + Retrieve a revision by ID + ~~~~~~~~~~~~~~~~~~~~~~~~~ + .. automethod:: cognite.client._api.three_d.ThreeDRevealAPI.retrieve_revision + + List sectors + ~~~~~~~~~~~~ + .. automethod:: cognite.client._api.three_d.ThreeDRevealAPI.list_sectors + + List nodes + ~~~~~~~~~~ + .. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.list_nodes + + List ancestor nodes + ~~~~~~~~~~~~~~~~~~~ + .. automethod:: cognite.client._api.three_d.ThreeDRevisionsAPI.list_ancestor_nodes + +Data classes +^^^^^^^^^^^^ +.. automodule:: cognite.client.data_classes.three_d :members: - :exclude-members: SchedulesClient - :undoc-members: :show-inheritance: - :inherited-members: -Time Series ------------ -Client + +Identity and access management +------------------------------ +Service accounts +^^^^^^^^^^^^^^^^ +List service accounts +~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.ServiceAccountsAPI.list + +Create service accounts +~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.ServiceAccountsAPI.create + +Delete service accounts +~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.ServiceAccountsAPI.delete + + +API keys +^^^^^^^^ +List API keys +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.APIKeysAPI.list + +Create API keys +~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.APIKeysAPI.create + +Delete API keys +~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.APIKeysAPI.delete + + +Groups ^^^^^^ -.. autoclass:: cognite.client.experimental.time_series.TimeSeriesClient - :members: +List groups +~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.list + +Create groups +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.create + +Delete groups +~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.delete + +List service accounts in a group +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.list_service_accounts + +Add service accounts to a group +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.add_service_account + +Remove service accounts from a group +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.GroupsAPI.remove_service_account + + +Security categories +^^^^^^^^^^^^^^^^^^^ +List security categories +~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.SecurityCategoriesAPI.list + +Create security categories +~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.SecurityCategoriesAPI.create + +Delete security categories +~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automethod:: cognite.client._api.iam.SecurityCategoriesAPI.delete + -Data Objects +Data classes ^^^^^^^^^^^^ -.. automodule:: cognite.client.experimental.time_series +.. automodule:: cognite.client.data_classes.iam :members: - :exclude-members: TimeSeriesClient - :undoc-members: :show-inheritance: - :inherited-members: -Datapoints ----------- -Client -^^^^^^ -.. autoclass:: cognite.client.experimental.datapoints.DatapointsClient + +Base data classes +----------------- +CogniteResource +^^^^^^^^^^^^^^^ +.. autoclass:: cognite.client.data_classes._base.CogniteResource :members: -Data Objects -^^^^^^^^^^^^ -.. automodule:: cognite.client.experimental.datapoints +CogniteResourceList +^^^^^^^^^^^^^^^^^^^ +.. autoclass:: cognite.client.data_classes._base.CogniteResourceList :members: - :exclude-members: DatapointsClient - :undoc-members: - :show-inheritance: - :inherited-members: -Sequences ----------- -Client -^^^^^^ -.. autoclass:: cognite.client.experimental.sequences.SequencesClient +CogniteResponse +^^^^^^^^^^^^^^^ +.. autoclass:: cognite.client.data_classes._base.CogniteResponse :members: -Data Objects -^^^^^^^^^^^^ -.. automodule:: cognite.client.experimental.sequences +CogniteFilter +^^^^^^^^^^^^^ +.. autoclass:: cognite.client.data_classes._base.CogniteFilter :members: - :exclude-members: SequencesClient - :undoc-members: - :show-inheritance: - :inherited-members: + +CogniteUpdate +^^^^^^^^^^^^^ +.. autoclass:: cognite.client.data_classes._base.CogniteUpdate + :members: + +Exceptions +---------- +CogniteAPIError +^^^^^^^^^^^^^^^ +.. autoexception:: cognite.client.exceptions.CogniteAPIError + +CogniteNotFoundError +^^^^^^^^^^^^^^^^^^^^ +.. autoexception:: cognite.client.exceptions.CogniteNotFoundError + +CogniteAPIKeyError +^^^^^^^^^^^^^^^^^^ +.. autoexception:: cognite.client.exceptions.CogniteAPIKeyError + +CogniteImportError +^^^^^^^^^^^^^^^^^^ +.. autoexception:: cognite.client.exceptions.CogniteImportError + +CogniteMissingClientError +^^^^^^^^^^^^^^^^^^^^^^^^^ +.. autoexception:: cognite.client.exceptions.CogniteMissingClientError + +Utils +----- +Convert timestamp to milliseconds since epoch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. autofunction:: cognite.client.utils.timestamp_to_ms + +Convert milliseconds since epoch to datetime +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. autofunction:: cognite.client.utils.ms_to_datetime diff --git a/docs/source/conf.py b/docs/source/conf.py index 4e2a49b04e..8ef564add3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- + # # cognite-sdk-python documentation build configuration file, created by # sphinx-quickstart on Mon Jan 8 14:36:34 2018. diff --git a/docs/source/index.rst b/docs/source/index.rst index fc436a2676..2c81f40901 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,16 +6,35 @@ Cognite Python SDK Documentation ================================ -Python SDK to ensure excellent user experience for developers and data scientists working with the Cognite Data Platform. +This is the Cognite Python SDK for developers and data scientists working with Cognite Data Fusion (CDF). The package is tightly integrated with pandas, and helps you work easily and efficiently with data in Cognite Data Fusion (CDF). -To install this package run the following command +.. contents:: + :local: + +Installation +^^^^^^^^^^^^ +.. note:: + + The SDK version 1.0.0 is currently in alpha so you need to include the :code:`--pre` flag in order to install it. + If you already have the the SDK installed in your environment, include the :code:`-U` flag to upgrade aswell. + +To install this package: .. code-block:: bash - pip install cognite-sdk + pip install --pre cognite-sdk -.. toctree:: - :maxdepth: 2 - :caption: Contents +To install this package without the pandas and NumPy support: +.. code-block:: bash + + pip install --pre cognite-sdk-core + +Contents +^^^^^^^^ +.. toctree:: cognite + +Examples +^^^^^^^^ +For a collection of scripts and Jupyter Notebooks that explain how to perform various tasks in Cognite Data Fusion (CDF) using Python, see the GitHub repository `here `__. diff --git a/generate_from_spec.py b/generate_from_spec.py new file mode 100644 index 0000000000..a832192a14 --- /dev/null +++ b/generate_from_spec.py @@ -0,0 +1,33 @@ +import argparse +import os + +from openapi.generator import CodeGenerator + +DEFAULT_SPEC_URL = "https://storage.googleapis.com/cognitedata-api-docs/dist/v1.json" + + +def main(spec_url, spec_path): + codegen = CodeGenerator(spec_url, spec_path) + spec = codegen.open_api_spec + + print("=" * 100) + print("{}: {}".format(spec.info.title, spec.info.version)) + print(spec.info.description) + print("=" * 100) + + for root, dirs, files in os.walk("./cognite/client/data_classes"): + for file in files: + file_path = os.path.join(root, file) + if file_path.endswith(".py"): + print("* Generating code in {}".format(file_path)) + codegen.generate(file_path, file_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--url", help="The url or path to the spec you wish to generate from. Defaults to latest spec.") + parser.add_argument( + "--path", help="The url or path to the spec you wish to generate from. Defaults to latest spec." + ) + args = parser.parse_args() + main(args.url or DEFAULT_SPEC_URL, args.path) diff --git a/cognite/client/_auxiliary/__init__.py b/openapi/__init__.py similarity index 100% rename from cognite/client/_auxiliary/__init__.py rename to openapi/__init__.py diff --git a/openapi/generator.py b/openapi/generator.py new file mode 100644 index 0000000000..70c1ac452a --- /dev/null +++ b/openapi/generator.py @@ -0,0 +1,292 @@ +import os +import re +from collections import namedtuple + +from black import FileMode, format_str + +from openapi import utils +from openapi.openapi import OpenAPISpec +from openapi.utils import TYPE_MAPPING + +TO_EXCLUDE = ["project", "cursor"] +GEN_CLASS_PATTERN = "# GenClass: ([\S ]+)\s+class (\S+)\(.+\):(?:(?!# GenStop)[\s\S])+# GenStop" +GEN_UPDATE_CLASS_PATTERN = "# GenUpdateClass: (\S+)\s+class (\S+)\(.+\):(?:(?!# GenStop)[\s\S])+# GenStop" + +GenClassSegment = namedtuple("GenClassSegment", ["schema_names", "class_name"]) +GenUpdateClassSegment = namedtuple("GenUpdateClassSegment", ["schema_name", "class_name"]) + + +class ClassGenerator: + def __init__(self, spec, input): + self._spec = spec + self._input = input + self.gen_class_segments = [GenClassSegment(*args) for args in re.findall(GEN_CLASS_PATTERN, self._input)] + + def generate_code_for_class_segments(self): + generated_segments = {} + for class_segment in self.gen_class_segments: + schema_names = class_segment.schema_names.split(", ") + schemas = [] + for schema_name in schema_names: + res = re.match("(.*)\.(.*)", schema_name) + if res is not None: + schema_name = res.group(1) + property_name = res.group(2) + schema = self._spec.components.schemas.get(schema_name) + property = self._get_schema_properties(schema)[property_name] + schemas.append(property) + else: + schemas.append(self._spec.components.schemas.get(schema_name)) + docstring = self.generate_docstring(schemas, indentation=4) + constructor_args = self.generate_constructor(schemas, indentation=4) + generated_segment = docstring + "\n" + constructor_args + generated_segments[class_segment.class_name] = generated_segment + return generated_segments + + def generate_docstring(self, schemas, indentation): + docstring = " " * indentation + '"""{}\n\n'.format(self._get_schema_description(schemas[0])) + docstring += " " * indentation + "Args:\n" + ignore = [p for p in TO_EXCLUDE] + for schema in schemas: + for prop_name, prop in self._get_schema_properties(schema).items(): + if prop_name not in ignore: + docstring += " " * (indentation + 4) + "{} ({}): {}\n".format( + utils.to_snake_case(prop_name), utils.get_type_hint(prop), self._get_schema_description(prop) + ) + ignore.append(prop_name) + docstring += ( + " " * (indentation + 4) + "cognite_client (CogniteClient): The client to associate with this object.\n" + ) + docstring += " " * indentation + '"""' + return docstring + + def generate_constructor(self, schemas, indentation): + constructor_params = [" " * indentation + "def __init__(self"] + ignore = [p for p in TO_EXCLUDE] + for schema in schemas: + for prop_name, prop in self._get_schema_properties(schema).items(): + prop_name = utils.to_snake_case(prop_name) + req = " = None" # TODO: check if prop is required or not + if prop_name not in ignore: + constructor_params.append("{}: {}{}".format(prop_name, utils.get_type_hint(prop), req)) + ignore.append(prop_name) + constructor_params = ", ".join(constructor_params) + ", cognite_client = None):" + constructor_body = "" + ignore = [p for p in TO_EXCLUDE] + for schema in schemas: + for prop_name, prop in self._get_schema_properties(schema).items(): + prop_name = utils.to_snake_case(prop_name) + if prop_name not in ignore: + constructor_body += " " * (indentation + 4) + "self.{} = {}\n".format(prop_name, prop_name) + ignore.append(prop_name) + constructor_body += " " * (indentation + 4) + "self._cognite_client = cognite_client\n" + return constructor_params + "\n" + constructor_body[:-1] + + @staticmethod + def _get_schema_description(schema): + if "allOf" in schema: + schema = schema["allOf"][0] + return schema.get("description", "No description.") + + @staticmethod + def _get_schema_properties(schema): + if "allOf" in schema: + properties = {} + for s in schema["allOf"]: + for prop_name, prop in s["properties"].items(): + properties[prop_name] = prop + return properties + return schema["properties"] + + +class UpdateClassGenerator: + def __init__(self, spec, input): + self._spec = spec + self._input = input + self.gen_update_class_segments = [ + GenUpdateClassSegment(*args) for args in re.findall(GEN_UPDATE_CLASS_PATTERN, self._input) + ] + + def generate_code_for_class_segments(self): + generated_segments = {} + for class_segment in self.gen_update_class_segments: + schema = self._spec.components.schemas.get(class_segment.schema_name) + docstring = self.generate_docstring(schema, indentation=4) + setters = self.generate_setters(schema, class_segment.class_name, indentation=4) + attr_update_classes = self.generate_attr_update_classes(class_segment.class_name) + generated_segment = docstring + "\n" + setters + "\n" + attr_update_classes + generated_segments[class_segment.class_name] = generated_segment + return generated_segments + + def generate_docstring(self, schema, indentation): + docstring = " " * indentation + '"""{}\n\n'.format(self._get_schema_description(schema)) + docstring += " " * indentation + "Args:\n" + for prop_name, prop in self._get_schema_properties(schema).items(): + indent = " " * (indentation + 4) + if prop_name == "id": + docstring += indent + "id (int): {}\n".format(self._get_schema_description(prop)) + elif prop_name == "externalId": + docstring += indent + "external_id (str): {}\n".format(self._get_schema_description(prop)) + docstring += " " * indentation + '"""' + return docstring + + def generate_setters(self, schema, class_name, indentation): + setters = [] + schema_properties = self._get_schema_properties(schema) + if "update" in schema_properties: + update_schema = schema_properties["update"] + else: + update_schema = schema + indent = " " * indentation + for prop_name, prop in self._get_schema_properties(update_schema).items(): + if prop_name == "id": + continue + update_prop_type_hints = {p: type_hint for p, type_hint in self._get_update_properties(prop)} + if "set" in update_prop_type_hints: + setter = indent + "@property\n" + setter += indent + "def {}(self):\n".format(utils.to_snake_case(prop_name)) + if update_prop_type_hints["set"] == "List": + setter += indent + indent + "return _List{}(self, '{}')".format(class_name, prop_name) + elif update_prop_type_hints["set"] == "Dict[str, Any]": + setter += indent + indent + "return _Object{}(self, '{}')".format(class_name, prop_name) + else: + setter += indent + indent + "return _Primitive{}(self, '{}')".format(class_name, prop_name) + setters.append(setter) + return "\n\n".join(setters) + + def generate_attr_update_classes(self, class_name): + update_classes = [] + update_class_methods = { + "Primitive": [("set", "Any")], + "Object": [("set", "Dict"), ("add", "Dict"), ("remove", "List")], + "List": [("set", "List"), ("add", "List"), ("remove", "List")], + } + indent = " " * 4 + for update_class_name, methods in update_class_methods.items(): + update_methods = [] + for method, value_type in methods: + update_method = indent + "def {}(self, value: {}) -> {}:\n".format(method, value_type, class_name) + update_method += indent + indent + "return self._{}(value)".format(method) + update_methods.append(update_method) + full_update_class_name = "_{}{}".format(update_class_name, class_name) + update_class = "class {}(Cognite{}Update):\n{}".format( + full_update_class_name, update_class_name, "\n\n".join(update_methods) + ) + update_classes.append(update_class) + return "\n\n".join(update_classes) + + @staticmethod + def _get_update_properties(property_update_schema): + update_properties = [] + if "oneOf" in property_update_schema: + update_objects = property_update_schema["oneOf"] + for update_obj in update_objects: + update_properties.extend(UpdateClassGenerator._get_update_properties(update_obj)) + return update_properties + for property_name, property in property_update_schema["properties"].items(): + if property_name != "setNull": + update_properties.append((property_name, TYPE_MAPPING[property["type"]])) + return update_properties + + @staticmethod + def _get_schema_description(schema): + if "allOf" in schema: + schema = schema["allOf"][0] + elif "oneOf" in schema: + return UpdateClassGenerator._get_schema_description(schema["oneOf"][0]) + return schema.get("description", "No description.") + + @staticmethod + def _get_schema_properties(schema): + if "allOf" in schema: + properties = {} + for s in schema["allOf"]: + for prop_name, prop in UpdateClassGenerator._get_schema_properties(s).items(): + properties[prop_name] = prop + return properties + if "oneOf" in schema: + assert len(schema["oneOf"]) <= 2, "oneOf contains {} schemas, expected 1 or 2".format(len(schema["oneOf"])) + first_schema_properties = UpdateClassGenerator._get_schema_properties(schema["oneOf"][0]) + if len(schema["oneOf"]) == 1: + return first_schema_properties + second_schema_properties = UpdateClassGenerator._get_schema_properties(schema["oneOf"][1]) + diff = list(set(first_schema_properties) - set(second_schema_properties)) + assert diff == ["id"] or diff == ["externalId"] + properties = {} + properties.update(first_schema_properties) + properties.update(second_schema_properties) + return properties + return schema["properties"] + + +class CodeGenerator: + def __init__(self, spec_url: str = None, spec_path: str = None): + self.open_api_spec = OpenAPISpec(url=spec_url, path=spec_path) + + def generate(self, input: str, output): + generated = self.generate_to_str(input) + with open(output, "w") as f: + f.write(generated) + + def generate_to_str(self, input: str): + input = self._parse_input(input) + class_generator = ClassGenerator(self.open_api_spec, input) + update_class_generator = UpdateClassGenerator(self.open_api_spec, input) + + content_with_generated_classes = self._generate_classes(input, class_generator) + content_with_generated_update_classes = self._generate_update_classes( + content_with_generated_classes, update_class_generator + ) + content_with_imports = self._generate_imports(content_with_generated_update_classes) + content_formatted = self._format_with_black(content_with_imports) + + return content_formatted + + def _generate_classes(self, content, class_generator): + generated_class_segments = class_generator.generate_code_for_class_segments() + for cls_name, code_segment in generated_class_segments.items(): + pattern = self._get_gen_class_replace_pattern(cls_name) + replace_with = self._get_gen_class_replace_string(cls_name, code_segment) + content = re.sub(pattern, replace_with, content) + return content + + def _generate_update_classes(self, content, method_generator): + generated_class_segments = method_generator.generate_code_for_class_segments() + for operation_id, code_segment in generated_class_segments.items(): + pattern = self._get_gen_update_class_replace_pattern(operation_id) + replace_with = self._get_gen_update_class_replace_string(operation_id, code_segment) + content = re.sub(pattern, replace_with, content) + return content + + def _generate_imports(self, content): + if re.search("from typing import \*", content) is None: + content = "from typing import *\n\n" + content + return content + + def _format_with_black(self, content): + return format_str(src_contents=content, mode=FileMode(line_length=120)) + + def _get_gen_class_replace_pattern(self, class_name): + return "# GenClass: ([\S ]+)\s+class {}\((.+)\):(?:(?!# GenStop)[\s\S])+# GenStop".format(class_name) + + def _get_gen_class_replace_string(self, class_name, code_segment): + return r"# GenClass: \1\nclass {}(\2):\n{}\n # GenStop".format(class_name, code_segment) + + def _get_gen_update_class_replace_pattern(self, class_name): + return "# GenUpdateClass: (\S+)\s+class {}\((.+)\):(?:(?!# GenStop)[\s\S])+# GenStop".format(class_name) + + def _get_gen_update_class_replace_string(self, class_name, code_segment): + return r"# GenUpdateClass: \1\nclass {}(\2):\n{}\n # GenStop".format(class_name, code_segment) + + @staticmethod + def _parse_input(input): + if re.match("^([^/]*/)*[^/]+\.py$", input): + if not os.path.isfile(input): + raise AssertionError("{} is not a python file or does not exist.".format(input)) + return CodeGenerator._read_file(input) + return input + + @staticmethod + def _read_file(path): + with open(path) as f: + return f.read() diff --git a/openapi/openapi.py b/openapi/openapi.py new file mode 100644 index 0000000000..5d8f270fde --- /dev/null +++ b/openapi/openapi.py @@ -0,0 +1,76 @@ +import json +import os +from subprocess import check_output +from tempfile import TemporaryDirectory + +import requests + +from openapi import utils + + +class Paths: + def __init__(self, paths): + self._paths = paths + + def get(self, path=None): + if path: + for curr_path, path_item in self._paths.items(): + if path == curr_path: + return path_item + ValueError("PathItem with path ´{}´ does not exist".format(path)) + return [path_item for _, path_item in self._paths.items()] + + def get_operation(self, operation_id: str): + for path_item in self.get(): + for method, operation in path_item.items(): + if "operationId" in operation and operation["operationId"] == operation_id: + return operation + raise ValueError("Operation with that id ´{}´ does not exist".format(operation_id)) + + +class Schemas: + def __init__(self, schemas): + self._schemas = schemas + + def get(self, name=None): + if name: + for schema_name, schema in self._schemas.items(): + if schema_name == name: + return schema + raise ValueError("Schema `{}` does not exist".format(name)) + return [schema for _, schema in self._schemas.items()] + + +class Components: + def __init__(self, components): + self._components = components + self.schemas = Schemas(self._components["schemas"]) + + +class Info: + def __init__(self, info): + self.title = info["title"] + self.version = info["version"] + self.description = utils.truncate_description(info["description"]) + + +class OpenAPISpec: + def __init__(self, url: str = None, path: str = None): + if url: + self._spec_url = url + self._spec = self.download_spec() + elif path: + with open(path) as f: + self._spec = json.load(f) + self.info = Info(self._spec["info"]) + self.components = Components(self._spec["components"]) + self.paths = Paths(self._spec["paths"]) + + def download_spec(self): + with TemporaryDirectory() as dir: + res = requests.get(self._spec_url).content + spec_path = os.path.join(dir, "spec.json") + with open(spec_path, "wb") as f: + f.write(res) + res = check_output("swagger-cli bundle -r {}".format(spec_path), shell=True) + return json.loads(res) diff --git a/cognite/client/_auxiliary/_protobuf_descriptors/__init__.py b/openapi/tests/__init__.py similarity index 100% rename from cognite/client/_auxiliary/_protobuf_descriptors/__init__.py rename to openapi/tests/__init__.py diff --git a/tests/test_client/__init__.py b/openapi/tests/input_output/__init__.py similarity index 100% rename from tests/test_client/__init__.py rename to openapi/tests/input_output/__init__.py diff --git a/openapi/tests/input_output/input.py b/openapi/tests/input_output/input.py new file mode 100644 index 0000000000..e67e08aade --- /dev/null +++ b/openapi/tests/input_output/input.py @@ -0,0 +1,41 @@ +class CogniteResource: + pass + + +class CogniteUpdate: + pass + + +class CogniteFilter: + pass + + +class CognitePrimitiveUpdate: + pass + + +class CogniteObjectUpdate: + pass + + +class CogniteListUpdate: + pass + + +# GenClass: Asset +class Asset(CogniteResource): + # GenStop + def to_pandas(self): + pass + + +# GenUpdateClass: AssetChange +class AssetUpdate(CogniteUpdate): + pass + # GenStop + + +# GenClass: AssetFilter.filter +class AssetFilter(CogniteFilter): + pass + # GenStop diff --git a/openapi/tests/input_output/output.py b/openapi/tests/input_output/output.py new file mode 100644 index 0000000000..f0d68da412 --- /dev/null +++ b/openapi/tests/input_output/output.py @@ -0,0 +1,177 @@ +from typing import * + + +class CogniteResource: + pass + + +class CogniteUpdate: + pass + + +class CogniteFilter: + pass + + +class CognitePrimitiveUpdate: + pass + + +class CogniteObjectUpdate: + pass + + +class CogniteListUpdate: + pass + + +# GenClass: Asset +class Asset(CogniteResource): + """Representation of a physical asset, e.g plant or piece of equipment + + Args: + external_id (str): External Id provided by client. Should be unique within the project. + name (str): Name of asset. Often referred to as tag. + parent_id (int): Javascript friendly internal ID given to the object. + description (str): Description of asset. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + id (int): Javascript friendly internal ID given to the object. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + path (List[int]): IDs of assets on the path to the asset. + depth (int): Asset path depth (number of levels below root node). + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + external_id: str = None, + name: str = None, + parent_id: int = None, + description: str = None, + metadata: Dict[str, Any] = None, + source: str = None, + id: int = None, + created_time: int = None, + last_updated_time: int = None, + path: List[int] = None, + depth: int = None, + cognite_client=None, + ): + self.external_id = external_id + self.name = name + self.parent_id = parent_id + self.description = description + self.metadata = metadata + self.source = source + self.id = id + self.created_time = created_time + self.last_updated_time = last_updated_time + self.path = path + self.depth = depth + self._cognite_client = cognite_client + + # GenStop + def to_pandas(self): + pass + + +# GenUpdateClass: AssetChange +class AssetUpdate(CogniteUpdate): + """Changes applied to asset + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project. + """ + + @property + def external_id(self): + return _PrimitiveAssetUpdate(self, "externalId") + + @property + def name(self): + return _PrimitiveAssetUpdate(self, "name") + + @property + def description(self): + return _PrimitiveAssetUpdate(self, "description") + + @property + def metadata(self): + return _ObjectAssetUpdate(self, "metadata") + + @property + def source(self): + return _PrimitiveAssetUpdate(self, "source") + + +class _PrimitiveAssetUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> AssetUpdate: + return self._set(value) + + +class _ObjectAssetUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> AssetUpdate: + return self._set(value) + + def add(self, value: Dict) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + +class _ListAssetUpdate(CogniteListUpdate): + def set(self, value: List) -> AssetUpdate: + return self._set(value) + + def add(self, value: List) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + # GenStop + + +# GenClass: AssetFilter.filter +class AssetFilter(CogniteFilter): + """No description. + + Args: + name (str): Name of asset. Often referred to as tag. + parent_ids (List[int]): No description. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + root (bool): filtered assets are root assets or not + external_id_prefix (str): External Id provided by client. Should be unique within the project. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + parent_ids: List[int] = None, + metadata: Dict[str, Any] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + root: bool = None, + external_id_prefix: str = None, + cognite_client=None, + ): + self.name = name + self.parent_ids = parent_ids + self.metadata = metadata + self.source = source + self.created_time = created_time + self.last_updated_time = last_updated_time + self.root = root + self.external_id_prefix = external_id_prefix + self._cognite_client = cognite_client + + # GenStop diff --git a/openapi/tests/input_output/output_test.py b/openapi/tests/input_output/output_test.py new file mode 100644 index 0000000000..f0d68da412 --- /dev/null +++ b/openapi/tests/input_output/output_test.py @@ -0,0 +1,177 @@ +from typing import * + + +class CogniteResource: + pass + + +class CogniteUpdate: + pass + + +class CogniteFilter: + pass + + +class CognitePrimitiveUpdate: + pass + + +class CogniteObjectUpdate: + pass + + +class CogniteListUpdate: + pass + + +# GenClass: Asset +class Asset(CogniteResource): + """Representation of a physical asset, e.g plant or piece of equipment + + Args: + external_id (str): External Id provided by client. Should be unique within the project. + name (str): Name of asset. Often referred to as tag. + parent_id (int): Javascript friendly internal ID given to the object. + description (str): Description of asset. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + id (int): Javascript friendly internal ID given to the object. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + path (List[int]): IDs of assets on the path to the asset. + depth (int): Asset path depth (number of levels below root node). + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + external_id: str = None, + name: str = None, + parent_id: int = None, + description: str = None, + metadata: Dict[str, Any] = None, + source: str = None, + id: int = None, + created_time: int = None, + last_updated_time: int = None, + path: List[int] = None, + depth: int = None, + cognite_client=None, + ): + self.external_id = external_id + self.name = name + self.parent_id = parent_id + self.description = description + self.metadata = metadata + self.source = source + self.id = id + self.created_time = created_time + self.last_updated_time = last_updated_time + self.path = path + self.depth = depth + self._cognite_client = cognite_client + + # GenStop + def to_pandas(self): + pass + + +# GenUpdateClass: AssetChange +class AssetUpdate(CogniteUpdate): + """Changes applied to asset + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project. + """ + + @property + def external_id(self): + return _PrimitiveAssetUpdate(self, "externalId") + + @property + def name(self): + return _PrimitiveAssetUpdate(self, "name") + + @property + def description(self): + return _PrimitiveAssetUpdate(self, "description") + + @property + def metadata(self): + return _ObjectAssetUpdate(self, "metadata") + + @property + def source(self): + return _PrimitiveAssetUpdate(self, "source") + + +class _PrimitiveAssetUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> AssetUpdate: + return self._set(value) + + +class _ObjectAssetUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> AssetUpdate: + return self._set(value) + + def add(self, value: Dict) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + +class _ListAssetUpdate(CogniteListUpdate): + def set(self, value: List) -> AssetUpdate: + return self._set(value) + + def add(self, value: List) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + + # GenStop + + +# GenClass: AssetFilter.filter +class AssetFilter(CogniteFilter): + """No description. + + Args: + name (str): Name of asset. Often referred to as tag. + parent_ids (List[int]): No description. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + created_time (Dict[str, Any]): Range between two timestamps + last_updated_time (Dict[str, Any]): Range between two timestamps + root (bool): filtered assets are root assets or not + external_id_prefix (str): External Id provided by client. Should be unique within the project. + cognite_client (CogniteClient): The client to associate with this object. + """ + + def __init__( + self, + name: str = None, + parent_ids: List[int] = None, + metadata: Dict[str, Any] = None, + source: str = None, + created_time: Dict[str, Any] = None, + last_updated_time: Dict[str, Any] = None, + root: bool = None, + external_id_prefix: str = None, + cognite_client=None, + ): + self.name = name + self.parent_ids = parent_ids + self.metadata = metadata + self.source = source + self.created_time = created_time + self.last_updated_time = last_updated_time + self.root = root + self.external_id_prefix = external_id_prefix + self._cognite_client = cognite_client + + # GenStop diff --git a/openapi/tests/test_generator.py b/openapi/tests/test_generator.py new file mode 100644 index 0000000000..7c70ff0359 --- /dev/null +++ b/openapi/tests/test_generator.py @@ -0,0 +1,197 @@ +import os + +from openapi.generator import ClassGenerator, CodeGenerator, UpdateClassGenerator + +input_path = os.path.join(os.path.dirname(__file__), "input_output/input.py") +output_path = os.path.join(os.path.dirname(__file__), "input_output/output.py") +output_test_path = os.path.join(os.path.dirname(__file__), "input_output/output_test.py") +with open(input_path) as f: + INPUT = f.read() +with open(output_path) as f: + OUTPUT = f.read() + +if os.getenv("CI") == "1": + CODE_GENERATOR = CodeGenerator(spec_path="deref-spec.json") +else: + CODE_GENERATOR = CodeGenerator(spec_url="https://storage.googleapis.com/cognitedata-api-docs/dist/v1.json") + + +class TestCodeGenerator: + def test_generated_output(self): + CODE_GENERATOR.generate(input_path, output_test_path) + with open(output_test_path) as f: + output = f.read() + assert OUTPUT == output + + def test_output_unchanged_if_regenerated(self): + output = CODE_GENERATOR.generate_to_str(output_path) + assert OUTPUT == output + + +CLASS_GENERATOR = ClassGenerator(CODE_GENERATOR.open_api_spec, INPUT) + + +class TestClassGenerator: + def test_get_gen_class_segments(self): + segments = CLASS_GENERATOR.gen_class_segments + assert ("Asset", "Asset") == segments[0] + assert ("AssetFilter.filter", "AssetFilter") == segments[1] + + assert "Asset" == segments[0].class_name + assert "Asset" == segments[0].schema_names + + def test_generate_docstring_from_schema(self): + schemas = [CLASS_GENERATOR._spec.components.schemas.get("Asset")] + docstring = CLASS_GENERATOR.generate_docstring(schemas, 4) + assert ( + """ \"\"\"Representation of a physical asset, e.g plant or piece of equipment + + Args: + external_id (str): External Id provided by client. Should be unique within the project. + name (str): Name of asset. Often referred to as tag. + parent_id (int): Javascript friendly internal ID given to the object. + description (str): Description of asset. + metadata (Dict[str, Any]): Custom, application specific metadata. String key -> String value + source (str): The source of this asset + id (int): Javascript friendly internal ID given to the object. + created_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + last_updated_time (int): It is the number of seconds that have elapsed since 00:00:00 Thursday, 1 January 1970, Coordinated Universal Time (UTC), minus leap seconds. + path (List[int]): IDs of assets on the path to the asset. + depth (int): Asset path depth (number of levels below root node). + cognite_client (CogniteClient): The client to associate with this object. + \"\"\"""" + == docstring + ) + + def test_generate_constructor_from_schema(self): + schemas = [CLASS_GENERATOR._spec.components.schemas.get("Asset")] + constructor = CLASS_GENERATOR.generate_constructor(schemas, indentation=4) + assert ( + """ def __init__(self, external_id: str = None, name: str = None, parent_id: int = None, description: str = None, metadata: Dict[str, Any] = None, source: str = None, id: int = None, created_time: int = None, last_updated_time: int = None, path: List[int] = None, depth: int = None, cognite_client = None): + self.external_id = external_id + self.name = name + self.parent_id = parent_id + self.description = description + self.metadata = metadata + self.source = source + self.id = id + self.created_time = created_time + self.last_updated_time = last_updated_time + self.path = path + self.depth = depth + self._cognite_client = cognite_client""" + == constructor + ) + + def test_generate_code_for_class_segments(self): + class_segments = CLASS_GENERATOR.generate_code_for_class_segments() + schemas = [CLASS_GENERATOR._spec.components.schemas.get("Asset")] + docstring = CLASS_GENERATOR.generate_docstring(schemas, indentation=4) + constructor = CLASS_GENERATOR.generate_constructor(schemas, indentation=4) + assert class_segments["Asset"] == docstring + "\n" + constructor + + +UPDATE_CLASS_GENERATOR = UpdateClassGenerator(CODE_GENERATOR.open_api_spec, INPUT) + + +class TestUpdateClassGenerator: + def test_get_gen_method_segments(self): + segments = UPDATE_CLASS_GENERATOR.gen_update_class_segments + assert ("AssetChange", "AssetUpdate") == segments[0] + assert "AssetUpdate" == segments[0].class_name + assert "AssetChange" == segments[0].schema_name + + def test_gen_docstring(self): + docstring = UPDATE_CLASS_GENERATOR.generate_docstring( + CLASS_GENERATOR._spec.components.schemas.get("AssetChange"), indentation=4 + ) + assert ( + """ \"\"\"Changes applied to asset + + Args: + id (int): Javascript friendly internal ID given to the object. + external_id (str): External Id provided by client. Should be unique within the project. + \"\"\"""" + == docstring + ) + + def test_gen_setters(self): + setters = UPDATE_CLASS_GENERATOR.generate_setters( + CLASS_GENERATOR._spec.components.schemas.get("EventChange"), "EventUpdate", indentation=4 + ) + assert ( + """ @property + def external_id(self): + return _PrimitiveEventUpdate(self, 'externalId') + + @property + def start_time(self): + return _PrimitiveEventUpdate(self, 'startTime') + + @property + def end_time(self): + return _PrimitiveEventUpdate(self, 'endTime') + + @property + def description(self): + return _PrimitiveEventUpdate(self, 'description') + + @property + def metadata(self): + return _ObjectEventUpdate(self, 'metadata') + + @property + def asset_ids(self): + return _ListEventUpdate(self, 'assetIds') + + @property + def source(self): + return _PrimitiveEventUpdate(self, 'source') + + @property + def type(self): + return _PrimitiveEventUpdate(self, 'type') + + @property + def subtype(self): + return _PrimitiveEventUpdate(self, 'subtype')""" + == setters + ) + + def test_generate_attr_update_classes(self): + attr_update_classes = UPDATE_CLASS_GENERATOR.generate_attr_update_classes("AssetUpdate") + assert ( + """class _PrimitiveAssetUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> AssetUpdate: + return self._set(value) + +class _ObjectAssetUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> AssetUpdate: + return self._set(value) + + def add(self, value: Dict) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value) + +class _ListAssetUpdate(CogniteListUpdate): + def set(self, value: List) -> AssetUpdate: + return self._set(value) + + def add(self, value: List) -> AssetUpdate: + return self._add(value) + + def remove(self, value: List) -> AssetUpdate: + return self._remove(value)""" + == attr_update_classes + ) + + def test_generate_code(self): + schema = UPDATE_CLASS_GENERATOR._spec.components.schemas.get("AssetChange") + docstring = UPDATE_CLASS_GENERATOR.generate_docstring(schema, indentation=4) + setters = UPDATE_CLASS_GENERATOR.generate_setters(schema, "AssetUpdate", indentation=4) + attr_update_classes = UPDATE_CLASS_GENERATOR.generate_attr_update_classes("AssetUpdate") + + generated = UPDATE_CLASS_GENERATOR.generate_code_for_class_segments()["AssetUpdate"] + assert generated == docstring + "\n" + setters + "\n" + attr_update_classes diff --git a/openapi/utils.py b/openapi/utils.py new file mode 100644 index 0000000000..60eacdda1e --- /dev/null +++ b/openapi/utils.py @@ -0,0 +1,51 @@ +import re + +TYPE_MAPPING = { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", + None: "None", + "None": "None", + "array": "List", + "object": "Dict[str, Any]", +} + + +def truncate_description(description): + max_len = 80 + lines = [] + line = [] + for word in description.split(): + line_so_far = " ".join(line) + if len(line_so_far) >= max_len: + lines.append(line_so_far) + line = [] + line.append(word) + lines.append(" ".join(line)) + return "\n".join(lines) + + +def get_type_hint(item): + if hasattr(item, "type"): + type = item.type + elif "type" in item: + type = item["type"] + elif "oneOf" in item and "type" in item["oneOf"][0]: + types = [] + for current in item["oneOf"]: + types.append(get_type_hint(current)) + return "Union[{}]".format(", ".join(types)) + + if type == "array": + return "List[{}]".format(get_type_hint(item["items"])) + elif type == "object": + return "Dict[str, Any]" + elif type in TYPE_MAPPING: + return TYPE_MAPPING[type] + raise "Unrecognized type '{}'".format(type) + + +def to_snake_case(str): + s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", str) + return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower() diff --git a/setup-core.py b/setup-core.py new file mode 100644 index 0000000000..3818d5c55a --- /dev/null +++ b/setup-core.py @@ -0,0 +1,20 @@ +import re + +from setuptools import find_packages, setup + +version = re.search('^__version__\s*=\s*"(.*)"', open("cognite/client/__init__.py").read(), re.M).group(1) + +setup( + name="cognite-sdk-core", + version=version, + description="Cognite API SDK for Python", + url="http://cognite-sdk-python.readthedocs.io/", + download_url="https://github.com/cognitedata/cognite-sdk-python/archive/{}.tar.gz".format(version), + author="Erlend Vollset", + author_email="erlend.vollset@cognite.com", + install_requires=["requests>=2.21.0,<3.0.0"], + python_requires=">=3.5", + packages=["cognite." + p for p in find_packages(where="cognite")], + zip_safe=False, + include_package_data=True, +) diff --git a/setup.py b/setup.py index 4229115c47..3cfc2a6da7 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ download_url="https://github.com/cognitedata/cognite-sdk-python/archive/{}.tar.gz".format(version), author="Erlend Vollset", author_email="erlend.vollset@cognite.com", - install_requires=["requests", "pandas", "protobuf", "cognite-logger==0.4.*"], + install_requires=["requests>=2.21.0,<3.0.0", "pandas"], python_requires=">=3.5", packages=["cognite." + p for p in find_packages(where="cognite")], zip_safe=False, diff --git a/tests/conftest.py b/tests/conftest.py index be4ab0a591..e17b31232f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,133 +1,81 @@ -import datetime -import json -import logging import os -import random -import string -from datetime import datetime from unittest import mock -from unittest.mock import MagicMock import pytest -from requests.structures import CaseInsensitiveDict +import responses from cognite.client import CogniteClient -from cognite.client.exceptions import APIError -from cognite.client.stable.datapoints import Datapoint -from cognite.client.stable.time_series import TimeSeries - -log = logging.getLogger(__name__) - -TEST_TS_START = 1514761200000 -TEST_TS_END = 1544828400000 -TEST_TS_MID = int((TEST_TS_START + TEST_TS_END) / 2) -TEST_TS_REASONABLE_INTERVAL = {"start": TEST_TS_MID - 2000000, "end": TEST_TS_MID + 2000000} -TEST_TS_REASONABLE_INTERVAL_DATETIME = { - "start": datetime.fromtimestamp((TEST_TS_MID - 600000) / 1000), - "end": datetime.fromtimestamp((TEST_TS_MID + 600000) / 1000), -} -TEST_TS_1_NAME = "SDK_TEST_TS_1_DO_NOT_DELETE" -TEST_TS_2_NAME = "SDK_TEST_TS_2_DO_NOT_DELETE" -TEST_TS_1_ID = None -TEST_TS_2_ID = None - - -@pytest.fixture(scope="session", autouse=True) -def time_series_in_cdp(): - global TEST_TS_1_ID, TEST_TS_2_ID - client = CogniteClient() - - try: - ts_list = [TimeSeries(name=TEST_TS_1_NAME)] - client.time_series.post_time_series(ts_list) - log.warning("Posted sdk test time series 1") - client.datapoints.post_datapoints( - name=TEST_TS_1_NAME, - datapoints=[Datapoint(timestamp=i, value=i) for i in range(TEST_TS_START, TEST_TS_END, int(3.6e6))], - ) - log.warning("Posted datapoints to sdk test time series 1") - TEST_TS_1_ID = client.time_series.get_time_series(prefix=TEST_TS_1_NAME).to_json()[0]["id"] - except APIError as e: - log.warning("Posting test time series 1 failed with code {}".format(e.code)) - - try: - ts_list = [TimeSeries(name=TEST_TS_2_NAME)] - client.time_series.post_time_series(ts_list) - log.warning("Posted sdk test time series 2") - client.datapoints.post_datapoints( - name=TEST_TS_2_NAME, - datapoints=[Datapoint(timestamp=i, value=i) for i in range(TEST_TS_START, TEST_TS_END, int(3.6e6))], - ) - log.warning("Posted datapoints to sdk test time series 2") - TEST_TS_2_ID = client.time_series.get_time_series(prefix=TEST_TS_2_NAME).to_json()[0]["id"] - except APIError as e: - log.warning("Posting test time series 2 failed with code {}".format(e.code)) - - TEST_TS_1_ID = client.time_series.get_time_series(prefix=TEST_TS_1_NAME).to_json()[0]["id"] - TEST_TS_2_ID = client.time_series.get_time_series(prefix=TEST_TS_2_NAME).to_json()[0]["id"] - yield TEST_TS_1_ID, TEST_TS_2_ID +from cognite.client._api.assets import AssetsAPI +from cognite.client._api.datapoints import DatapointsAPI +from cognite.client._api.events import EventsAPI +from cognite.client._api.files import FilesAPI +from cognite.client._api.login import LoginAPI +from cognite.client._api.raw import RawAPI, RawDatabasesAPI, RawRowsAPI, RawTablesAPI +from cognite.client._api.time_series import TimeSeriesAPI +from tests.utils import BASE_URL @pytest.fixture -def disable_gzip(): - os.environ["COGNITE_DISABLE_GZIP"] = "1" - yield - del os.environ["COGNITE_DISABLE_GZIP"] - - -class MockRequest(mock.Mock): - def __init__(self): - super().__init__() - self.method = "GET" - self.url = "http://some.url" - self.headers = {} - - -class MockReturnValue(mock.Mock): - """Helper class for building mock request responses. +def client(rsps_with_login_mock): + yield CogniteClient() - Should be assigned to MagicMock.return_value - """ - def __init__( - self, status=200, content="CONTENT", json_data=None, raise_for_status=None, headers=CaseInsensitiveDict() - ): - mock.Mock.__init__(self) - if "X-Request-Id" not in headers: - headers["X-Request-Id"] = "1234567890" - - # mock raise_for_status call w/optional error - self.raise_for_status = mock.Mock() - if raise_for_status: - self.raise_for_status.side_effect = raise_for_status +@pytest.fixture +def rsps_with_login_mock(): + with responses.RequestsMock() as rsps: + rsps.add(responses.GET, "https://pypi.python.org/simple/cognite-sdk/#history", status=200, body="") + rsps.add( + responses.GET, + BASE_URL + "/login/status", + status=200, + json={"data": {"project": "test", "loggedIn": True, "user": "bla", "projectId": "bla", "apiKeyId": 1}}, + ) + yield rsps - # set status code and content - self.status_code = status - # requests.models.Response.ok mock - self.ok = status < 400 - self.content = content - self.headers = headers +@pytest.fixture +def mock_cognite_client(): + with mock.patch("cognite.client.CogniteClient") as client_mock: + cog_client_mock = mock.MagicMock(spec=CogniteClient) + cog_client_mock.time_series = mock.MagicMock(spec=TimeSeriesAPI) + cog_client_mock.datapoints = mock.MagicMock(spec=DatapointsAPI) + cog_client_mock.assets = mock.MagicMock(spec=AssetsAPI) + cog_client_mock.events = mock.MagicMock(spec=EventsAPI) + cog_client_mock.files = mock.MagicMock(spec=FilesAPI) + cog_client_mock.login = mock.MagicMock(spec=LoginAPI) + raw_mock = mock.MagicMock(spec=RawAPI) + raw_mock.databases = mock.MagicMock(spec=RawDatabasesAPI) + raw_mock.tables = mock.MagicMock(spec=RawTablesAPI) + raw_mock.rows = mock.MagicMock(spec=RawRowsAPI) + cog_client_mock.raw = raw_mock + client_mock.return_value = cog_client_mock + yield - # add json data if provided - if json_data: - self.json = lambda: json_data - self.request = MockRequest() +@pytest.fixture +def rsps(): + with responses.RequestsMock() as rsps: + yield rsps - self.raw = MagicMock() - def __setattr__(self, key, value): - if key == "_content": - self.json = lambda: json.loads(value.decode()) - super().__setattr__(key, value) +@pytest.fixture +def disable_gzip(): + os.environ["COGNITE_DISABLE_GZIP"] = "1" + yield + del os.environ["COGNITE_DISABLE_GZIP"] -def get_time_w_offset(**kwargs): - curr_time = datetime.datetime.now() - offset_time = curr_time - datetime.timedelta(**kwargs) - return int(round(offset_time.timestamp() * 1000)) +def pytest_addoption(parser): + parser.addoption( + "--test-deps-only-core", action="store_true", default=False, help="Test only core deps are installed" + ) -def generate_random_string(n): - return "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(n)) +def pytest_collection_modifyitems(config, items): + if config.getoption("--test-deps-only-core"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need ----test-deps-only-core option to run") + for item in items: + if "coredeps" in item.keywords: + item.add_marker(skip_slow) diff --git a/tests/test_client/test_api_client.py b/tests/test_client/test_api_client.py deleted file mode 100644 index 3f42d2b217..0000000000 --- a/tests/test_client/test_api_client.py +++ /dev/null @@ -1,300 +0,0 @@ -# -*- coding: utf-8 -*- -import gzip -import json -import re -from unittest import mock - -import pytest - -from cognite.client import APIError -from cognite.client._api_client import APIClient, _model_hosting_emulator_url_converter -from tests.conftest import MockReturnValue - -RESPONSE = { - "data": { - "items": [ - { - "id": 123456789, - "name": "a_name", - "parentId": 234567890, - "description": "A piece of equipment", - "metadata": {"md1": "some data"}, - } - ] - } -} - - -@pytest.fixture(autouse=True) -def api_client(): - client = APIClient( - project="test_proj", - base_url="http://localtest.com/api/0.5/projects/test_proj", - num_of_workers=1, - cookies={"a-cookie": "a-cookie-val"}, - headers={}, - timeout=60, - ) - yield client - - -@pytest.fixture(autouse=True) -def url(): - yield "/assets" - - -class TestRequests: - @mock.patch("requests.sessions.Session.delete") - def test_delete_request_ok(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(json_data=RESPONSE) - response = api_client._delete(url) - assert response.status_code == 200 - assert len(response.json()["data"]["items"]) == len(RESPONSE) - - @mock.patch("requests.sessions.Session.delete") - def test_delete_request_failed(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=400, json_data={"error": "Client error"}) - - with pytest.raises(APIError) as e: - api_client._delete(url) - assert re.match("Client error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, content="Server error") - - with pytest.raises(APIError) as e: - api_client._delete(url) - assert re.match("Server error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, json_data={"error": "Server error"}) - - with pytest.raises(APIError) as e: - api_client._delete(url) - assert re.match("Server error", str(e.value)) - - mock_request.return_value = MockReturnValue( - status=400, json_data={"error": {"code": 400, "message": "Client error"}} - ) - - with pytest.raises(APIError) as e: - api_client._delete(url) - assert re.match("Client error | code: 400 | X-Request-ID:", str(e.value)) - assert e.value.code == 400 - assert e.value.message == "Client error" - - @mock.patch("requests.sessions.Session.delete") - def test_delete_request_exception(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=500) - mock_request.side_effect = Exception("Custom error") - - with pytest.raises(Exception) as e: - api_client._delete(url) - assert re.match("Custom error", str(e.value)) - - @mock.patch("requests.sessions.Session.get") - def test_get_request(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(json_data=RESPONSE) - response = api_client._get(url) - - assert response.status_code == 200 - assert len(response.json()["data"]["items"]) == len(RESPONSE) - - @mock.patch("requests.sessions.Session.get") - def test_get_request_failed(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=400, json_data={"error": "Client error"}) - - with pytest.raises(APIError) as e: - api_client._get(url) - assert re.match("Client error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, content="Server error") - - with pytest.raises(APIError) as e: - api_client._get(url) - assert re.match("Server error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, json_data={"error": "Server error"}) - - with pytest.raises(APIError) as e: - api_client._get(url) - assert re.match("Server error", str(e.value)) - - @mock.patch("requests.sessions.Session.get") - def test_get_request_exception(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=500) - mock_request.side_effect = Exception("Custom error") - - with pytest.raises(Exception) as e: - api_client._get(url) - assert re.match("Custom error", str(e.value)) - - @mock.patch("requests.sessions.Session.get") - def test_get_request_with_autopaging(self, mock_request, api_client, url): - mock_request.side_effect = [ - MockReturnValue(json_data={"data": {"items": [1, 2, 3], "nextCursor": "next"}}), - MockReturnValue(json_data={"data": {"items": [4, 5, 6], "nextCursor": "next"}}), - MockReturnValue(json_data={"data": {"items": [7, 8, 9], "nextCursor": None}}), - ] - - res = api_client._get(url, params={}, autopaging=True) - assert mock_request.call_count == 3 - assert {"data": {"items": [1, 2, 3, 4, 5, 6, 7, 8, 9]}} == res.json() - - @mock.patch("requests.sessions.Session.post") - def test_post_request_ok(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(json_data=RESPONSE) - - response = api_client._post(url, RESPONSE) - response_json = response.json() - - assert response.status_code == 200 - assert len(response_json["data"]["items"]) == len(RESPONSE) - - @mock.patch("requests.sessions.Session.post") - def test_post_request_failed(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=400, json_data={"error": "Client error"}) - - with pytest.raises(APIError) as e: - api_client._post(url, RESPONSE) - assert re.match("Client error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, content="Server error") - - with pytest.raises(APIError) as e: - api_client._post(url, RESPONSE) - assert re.match("Server error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, json_data={"error": "Server error"}) - - with pytest.raises(APIError) as e: - api_client._post(url, RESPONSE) - assert re.match("Server error", str(e.value)) - - @mock.patch("requests.sessions.Session.post") - def test_post_request_exception(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=500) - mock_request.side_effect = Exception("Custom error") - - with pytest.raises(Exception) as e: - api_client._post(url, RESPONSE) - assert re.match("Custom error", str(e.value)) - - @mock.patch("requests.sessions.Session.post") - def test_post_request_args(self, mock_request, api_client, url): - def check_args_to_post_and_return_mock( - arg_url, data=None, headers=None, params=None, cookies=None, timeout=None - ): - # URL is sent as is - assert arg_url == api_client._base_url + url - - # cookies should be the same - assert cookies == {"a-cookie": "a-cookie-val"} - - # Return the mock response - return MockReturnValue(json_data=RESPONSE) - - mock_request.side_effect = check_args_to_post_and_return_mock - - response = api_client._post(url, RESPONSE, headers={"Existing-Header": "SomeValue"}) - - assert response.status_code == 200 - - @pytest.mark.usefixtures("disable_gzip") - @mock.patch("requests.sessions.Session.post") - def test_post_request_gzip_disabled(self, mock_request, api_client, url): - def check_gzip_disabled_and_return_mock( - arg_url, data=None, headers=None, params=None, cookies=None, timeout=None - ): - # URL is sent as is - assert arg_url == api_client._base_url + url - # gzip is not added as Content-Encoding header - assert "Content-Encoding" not in headers - # data is not gzipped. - assert len(json.loads(data)["data"]["items"]) == len(RESPONSE) - # Return the mock response - return MockReturnValue(json_data=RESPONSE) - - mock_request.side_effect = check_gzip_disabled_and_return_mock - - response = api_client._post(url, RESPONSE, headers={}) - assert response.status_code == 200 - - @mock.patch("requests.sessions.Session.put") - def test_put_request_ok(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(json_data=RESPONSE) - - response = api_client._put(url, body=RESPONSE) - response_json = response.json() - - assert response.status_code == 200 - assert len(response_json["data"]["items"]) == len(RESPONSE) - - @mock.patch("requests.sessions.Session.put") - def test_put_request_failed(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=400, json_data={"error": "Client error"}) - - with pytest.raises(APIError) as e: - api_client._put(url) - assert re.match("Client error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, content="Server error") - - with pytest.raises(APIError) as e: - api_client._put(url) - assert re.match("Server error", str(e.value)) - - mock_request.return_value = MockReturnValue(status=500, json_data={"error": "Server error"}) - - with pytest.raises(APIError) as e: - api_client._put(url) - assert re.match("Server error", str(e.value)) - - @mock.patch("requests.sessions.Session.put") - def test_put_request_exception(self, mock_request, api_client, url): - mock_request.return_value = MockReturnValue(status=500) - mock_request.side_effect = Exception("Custom error") - - with pytest.raises(Exception) as e: - api_client._put(url) - assert re.match("Custom error", str(e.value)) - - @mock.patch("requests.sessions.Session.put") - def test_put_request_args(self, mock_request, api_client, url): - import json - - def check_args_to_put_and_return_mock( - arg_url, data=None, headers=None, params=None, cookies=None, timeout=None - ): - # URL is sent as is - assert arg_url == api_client._base_url + url - # data is json encoded - assert len(json.loads(gzip.decompress(data).decode("utf8"))["data"]["items"]) == len(RESPONSE) - # cookies should be the same - assert cookies == {"a-cookie": "a-cookie-val"} - # Return the mock response - return MockReturnValue(json_data=RESPONSE) - - mock_request.side_effect = check_args_to_put_and_return_mock - - response = api_client._put(url, RESPONSE, headers={"Existing-Header": "SomeValue"}) - - assert response.status_code == 200 - - @pytest.mark.parametrize( - "input, expected", - [ - ( - "https://api.cognitedata.com/api/0.6/projects/test-project/analytics/models", - "http://localhost:8000/api/0.1/projects/test-project/models", - ), - ( - "https://api.cognitedata.com/api/0.6/projects/test-project/analytics/models/sourcepackages/1", - "http://localhost:8000/api/0.1/projects/test-project/models/sourcepackages/1", - ), - ( - "https://api.cognitedata.com/api/0.6/projects/test-project/assets/update", - "https://api.cognitedata.com/api/0.6/projects/test-project/assets/update", - ), - ], - ) - def test_nostromo_emulator_url_converter(self, input, expected): - assert expected == _model_hosting_emulator_url_converter(input) diff --git a/tests/test_client/test_cognite_client.py b/tests/test_client/test_cognite_client.py deleted file mode 100644 index 0f34968b73..0000000000 --- a/tests/test_client/test_cognite_client.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import random -import sys -import threading -import types -from multiprocessing.pool import ThreadPool -from time import sleep - -import pytest - -from cognite.client import APIError, CogniteClient - - -@pytest.fixture -def client(): - yield CogniteClient() - - -@pytest.fixture -def default_client_config(): - from cognite.client.cognite_client import DEFAULT_BASE_URL, DEFAULT_NUM_OF_WORKERS, DEFAULT_TIMEOUT - - yield DEFAULT_BASE_URL, DEFAULT_NUM_OF_WORKERS, DEFAULT_TIMEOUT - - -@pytest.fixture -def environment_client_config(): - base_url = "blabla" - num_of_workers = 1 - timeout = 10 - - os.environ["COGNITE_BASE_URL"] = base_url - os.environ["COGNITE_NUM_WORKERS"] = str(num_of_workers) - os.environ["COGNITE_TIMEOUT"] = str(timeout) - - yield base_url, num_of_workers, timeout - - del os.environ["COGNITE_BASE_URL"] - del os.environ["COGNITE_NUM_WORKERS"] - del os.environ["COGNITE_TIMEOUT"] - - -class TestCogniteClient: - def test_get(self, client): - res = client.get("/login/status") - assert res.status_code == 200 - - def test_post(self, client): - res = client.post("/login", body={"apiKey": client._CogniteClient__api_key}) - assert res.status_code == 200 - - def test_put(self, client): - with pytest.raises(APIError) as e: - client.put("/login") - assert e.value.code == 405 - - def test_delete(self, client): - with pytest.raises(APIError) as e: - client.delete("/login") - assert e.value.code == 405 - - def test_project_is_correct(self, client): - assert client._project == "mltest" - - def assert_config_is_correct(self, client, base_url, num_of_workers, timeout): - assert client._base_url == base_url - assert type(client._base_url) is str - - assert client._num_of_workers == num_of_workers - assert type(client._num_of_workers) is int - - assert client._timeout == timeout - assert type(client._timeout) is int - - def test_default_config(self, client, default_client_config): - self.assert_config_is_correct(client, *default_client_config) - - def test_parameter_config(self): - base_url = "blabla" - num_of_workers = 1 - timeout = 10 - - client = CogniteClient(project="something", base_url=base_url, num_of_workers=num_of_workers, timeout=timeout) - self.assert_config_is_correct(client, base_url, num_of_workers, timeout) - - def test_environment_config(self, environment_client_config): - client = CogniteClient(project="something") - self.assert_config_is_correct(client, *environment_client_config) - - @pytest.fixture - def thread_local_credentials_module(self): - credentials_module = types.ModuleType("cognite._thread_local") - credentials_module.credentials = threading.local() - sys.modules["cognite._thread_local"] = credentials_module - yield - del sys.modules["cognite._thread_local"] - - def create_client_and_check_config(self, i): - from cognite._thread_local import credentials - - api_key = "thread-local-api-key{}".format(i) - project = "thread-local-project{}".format(i) - - credentials.api_key = api_key - credentials.project = project - - sleep(random.random()) - client = CogniteClient() - - assert api_key == client._CogniteClient__api_key - assert project == client._project - - def test_create_client_thread_local_config(self, thread_local_credentials_module): - with ThreadPool() as pool: - pool.map(self.create_client_and_check_config, list(range(16))) diff --git a/tests/test_client/test_sdk_utils.py b/tests/test_client/test_sdk_utils.py deleted file mode 100644 index d21ace2e53..0000000000 --- a/tests/test_client/test_sdk_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -from datetime import datetime - -import pytest - -import cognite.client._utils as utils -from cognite.client.stable.datapoints import Datapoint, TimeseriesWithDatapoints - - -class TestConversions: - def test_datetime_to_ms(self): - from datetime import datetime - - assert utils.datetime_to_ms(datetime(2018, 1, 31)) == 1517356800000 - assert utils.datetime_to_ms(datetime(2018, 1, 31, 11, 11, 11)) == 1517397071000 - assert utils.datetime_to_ms(datetime(100, 1, 31)) == -59008867200000 - with pytest.raises(AttributeError): - utils.datetime_to_ms(None) - - def test_round_to_nearest(self): - assert utils._round_to_nearest(12, 10) == 10 - assert utils._round_to_nearest(8, 10) == 10 - - def test_granularity_to_ms(self): - assert utils.granularity_to_ms("10s") == 10000 - assert utils.granularity_to_ms("10m") == 600000 - - def test_interval_to_ms(self): - assert isinstance(utils.interval_to_ms(None, None)[0], int) - assert isinstance(utils.interval_to_ms(None, None)[1], int) - assert isinstance(utils.interval_to_ms("1w-ago", "1d-ago")[0], int) - assert isinstance(utils.interval_to_ms("1w-ago", "1d-ago")[1], int) - assert isinstance(utils.interval_to_ms(datetime(2018, 2, 1), datetime(2018, 3, 1))[0], int) - assert isinstance(utils.interval_to_ms(datetime(2018, 2, 1), datetime(2018, 3, 1))[1], int) - - def test_time_ago_to_ms(self): - assert utils._time_ago_to_ms("3w-ago") == 1814400000 - assert utils._time_ago_to_ms("1d-ago") == 86400000 - assert utils._time_ago_to_ms("1s-ago") == 1000 - assert utils - assert utils._time_ago_to_ms("not_correctly_formatted") is None - - -class TestFirstFit: - def test_with_timeserieswithdatapoints(self): - from typing import List - - timeseries_with_100_datapoints = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(100)] - ) - timeseries_with_200_datapoints = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(200)] - ) - timeseries_with_300_datapoints = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(300)] - ) - - all_timeseries = [ - timeseries_with_100_datapoints, - timeseries_with_200_datapoints, - timeseries_with_300_datapoints, - ] - - result = utils.first_fit(list_items=all_timeseries, max_size=300, get_count=lambda x: len(x.datapoints)) - - assert len(result) == 2 - - -class TestMisc: - @pytest.mark.parametrize( - "start, end, granularity, num_of_workers, expected_output", - [ - (1550241236999, 1550244237001, "1d", 1, [{"start": 1550241236999, "end": 1550244237001}]), - (0, 10000, "1s", 10, [{"start": i, "end": i + 1000} for i in range(0, 10000, 1000)]), - (0, 2500, "1s", 3, [{"start": 0, "end": 1000}, {"start": 1000, "end": 2500}]), - ], - ) - def test_get_datapoints_windows(self, start, end, granularity, num_of_workers, expected_output): - res = utils.get_datapoints_windows(start=start, end=end, granularity=granularity, num_of_workers=num_of_workers) - assert expected_output == res diff --git a/tests/test_experimental/source_package_for_tests/artifacts/artifact1.txt b/tests/test_experimental/source_package_for_tests/artifacts/artifact1.txt deleted file mode 100644 index d95f3ad14d..0000000000 --- a/tests/test_experimental/source_package_for_tests/artifacts/artifact1.txt +++ /dev/null @@ -1 +0,0 @@ -content diff --git a/tests/test_experimental/source_package_for_tests/artifacts/sub_dir/artifact2.txt b/tests/test_experimental/source_package_for_tests/artifacts/sub_dir/artifact2.txt deleted file mode 100644 index d95f3ad14d..0000000000 --- a/tests/test_experimental/source_package_for_tests/artifacts/sub_dir/artifact2.txt +++ /dev/null @@ -1 +0,0 @@ -content diff --git a/tests/test_experimental/source_package_for_tests/my_model/model.py b/tests/test_experimental/source_package_for_tests/my_model/model.py deleted file mode 100644 index f6221a6ec9..0000000000 --- a/tests/test_experimental/source_package_for_tests/my_model/model.py +++ /dev/null @@ -1,11 +0,0 @@ -class Model: - @staticmethod - def train(self): - pass - - @staticmethod - def load(self): - pass - - def predict(self): - pass diff --git a/tests/test_experimental/source_package_for_tests/setup.py b/tests/test_experimental/source_package_for_tests/setup.py deleted file mode 100644 index 21a7d79f4f..0000000000 --- a/tests/test_experimental/source_package_for_tests/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from setuptools import find_packages, setup - -setup(name="simple-model", version="0.1", packages=find_packages(), description="A simple model") diff --git a/tests/test_experimental/test_datapoints.py b/tests/test_experimental/test_datapoints.py deleted file mode 100644 index 2f7f1fa715..0000000000 --- a/tests/test_experimental/test_datapoints.py +++ /dev/null @@ -1,66 +0,0 @@ -from datetime import datetime -from random import randint - -import numpy as np -import pandas as pd -import pytest - -from cognite.client import CogniteClient -from cognite.client.experimental.datapoints import DatapointsClient, DatapointsResponse -from cognite.client.stable.time_series import TimeSeries -from tests.conftest import TEST_TS_REASONABLE_INTERVAL, TEST_TS_REASONABLE_INTERVAL_DATETIME - -cognite_client = CogniteClient() -datapoints = cognite_client.experimental.datapoints - -TS_NAME = None - -dps_params = [TEST_TS_REASONABLE_INTERVAL, TEST_TS_REASONABLE_INTERVAL_DATETIME] - - -@pytest.fixture(autouse=True, scope="class") -def ts_name(): - global TS_NAME - TS_NAME = "test_ts_{}".format(randint(1, 2 ** 53 - 1)) - - -@pytest.fixture(scope="class") -def datapoints_fixture(): - tso = TimeSeries(TS_NAME) - cognite_client.time_series.post_time_series([tso]) - yield - cognite_client.time_series.delete_time_series(TS_NAME) - - -@pytest.mark.usefixtures("datapoints_fixture") -class TestDatapoints: - @pytest.fixture(scope="class", params=dps_params) - def get_dps_response_obj(self, request, time_series_in_cdp): - yield datapoints.get_datapoints( - id=time_series_in_cdp[0], - start=request.param["start"], - end=request.param["end"], - include_outside_points=True, - ) - - def test_get_datapoints(self, get_dps_response_obj): - - assert isinstance(get_dps_response_obj, DatapointsResponse) - - def test_get_dps_output_formats(self, get_dps_response_obj): - assert isinstance(get_dps_response_obj.to_pandas(), pd.DataFrame) - assert isinstance(get_dps_response_obj.to_json(), dict) - - def test_get_dps_correctly_spaced(self, get_dps_response_obj): - timestamps = get_dps_response_obj.to_pandas().timestamp.values - deltas = np.diff(timestamps, 1) - assert (deltas != 0).all() - assert (deltas % 10000 == 0).all() - - def test_get_dps_with_limit(self, time_series_in_cdp): - res = datapoints.get_datapoints(id=time_series_in_cdp[0], start=0, limit=1) - assert len(res.to_json().get("datapoints")) == 1 - - def test_get_dps_with_end_now(self, time_series_in_cdp): - res = datapoints.get_datapoints(id=time_series_in_cdp[0], start=0, end="now", limit=100) - assert len(res.to_json().get("datapoints")) == 100 diff --git a/tests/test_experimental/test_modelhosting.py b/tests/test_experimental/test_modelhosting.py deleted file mode 100644 index e5d69cebd2..0000000000 --- a/tests/test_experimental/test_modelhosting.py +++ /dev/null @@ -1,480 +0,0 @@ -import gzip -import json -import os -import time -from random import randint -from unittest import mock - -import pytest - -from cognite.client import APIError, CogniteClient -from cognite.client.experimental.model_hosting.models import ( - ModelArtifactCollectionResponse, - ModelArtifactResponse, - ModelCollectionResponse, - ModelLogResponse, - ModelResponse, - ModelVersionCollectionResponse, - ModelVersionResponse, - PredictionError, -) -from cognite.client.experimental.model_hosting.schedules import ScheduleCollectionResponse, ScheduleResponse -from cognite.client.experimental.model_hosting.source_packages import ( - SourcePackageCollectionResponse, - SourcePackageResponse, -) -from tests.conftest import MockReturnValue -from tests.utils import get_call_args_data_from_mock - -modelhosting_client = CogniteClient().experimental.model_hosting - -models = modelhosting_client.models -schedules = modelhosting_client.schedules -source_packages = modelhosting_client.source_packages - - -@pytest.fixture -def mock_data_spec(): - class FakeScheduleDataSpec: - def dump(self): - return {"spec": "spec"} - - return FakeScheduleDataSpec() - - -class TestSourcePackages: - @pytest.fixture(scope="class") - def source_package_file_path(self): - file_path = "/tmp/sp.tar.gz" - with open(file_path, "w") as f: - f.write("content") - yield file_path - os.remove(file_path) - - @pytest.fixture(scope="class") - def created_source_package(self, source_package_file_path): - sp_name = "test-sp-{}".format(randint(0, 1e5)) - sp = source_packages.upload_source_package( - name=sp_name, - package_name="whatever", - available_operations=["TRAIN", "PREDICT"], - runtime_version="0.1", - file_path=source_package_file_path, - ) - assert sp.upload_url is None - yield sp - source_packages.delete_source_package(id=sp.id) - - @pytest.fixture(scope="class") - def source_package_directory(self): - yield os.path.join(os.path.dirname(__file__), "source_package_for_tests") - - def test_build_and_create_source_package(self, source_package_directory): - sp_name = "test-sp-{}".format(randint(0, 1e5)) - sp = source_packages.build_and_upload_source_package( - name=sp_name, runtime_version="0.1", package_directory=source_package_directory - ) - assert sp.upload_url is None - - sp = source_packages.get_source_package(sp.id) - assert ["TRAIN", "PREDICT"] == sp.available_operations - assert "my_model" == sp.package_name - source_packages.delete_source_package(id=sp.id) - - def test_list_source_packages(self, created_source_package): - res = source_packages.list_source_packages() - assert len(res) > 0 - assert isinstance(res, SourcePackageCollectionResponse) - assert isinstance(res[:1], SourcePackageCollectionResponse) - assert isinstance(res[0], SourcePackageResponse) - for sp in res: - assert isinstance(sp, SourcePackageResponse) - - def test_get_source_package(self, created_source_package): - for i in range(5): - sp = source_packages.get_source_package(created_source_package.id) - if sp.is_uploaded: - break - time.sleep(1) - assert isinstance(sp, SourcePackageResponse) - assert sp.id == created_source_package.id - assert sp.is_uploaded is True - - def test_deprecate_source_package(self, created_source_package): - sp = source_packages.deprecate_source_package(created_source_package.id) - assert sp.is_deprecated is True - sp = source_packages.get_source_package(created_source_package.id) - assert sp.is_deprecated is True - - def test_download_code(self, created_source_package): - source_packages.download_source_package_code(id=created_source_package.id, directory=os.getcwd()) - sp_name = source_packages.get_source_package(id=created_source_package.id).name - file_path = os.path.join(os.getcwd(), sp_name + ".tar.gz") - assert os.path.isfile(file_path) - with open(file_path) as f: - assert "content" == f.read() - os.remove(file_path) - - @pytest.mark.skip( - reason="Deleting source package code currently breaks deleting the source package, " - "and causes get_source_package_code to return 500 instead of 404." - ) - def test_delete_code(self, created_source_package): - source_packages.delete_source_package_code(id=created_source_package.id) - with pytest.raises(APIError, match="deleted"): - source_packages.download_source_package_code(id=created_source_package.id) - - -class TestModels: - @pytest.fixture - def created_model(self): - model_name = "test-model-{}".format(randint(0, 1e5)) - model = models.create_model(name=model_name) - yield model - models.delete_model(model.id) - - def test_get_model(self, created_model): - model = models.get_model(created_model.id) - assert model.name == created_model.name - assert model.active_version_id is None - - def test_list_models(self, created_model): - res = models.list_models() - assert len(res) > 0 - assert isinstance(res, ModelCollectionResponse) - assert isinstance(res[:1], ModelCollectionResponse) - assert isinstance(res[0], ModelResponse) - for model in res: - assert isinstance(model, ModelResponse) - - def test_update_model(self, created_model): - res = models.update_model(id=created_model.id, description="bla") - assert isinstance(res, ModelResponse) - assert res.description == "bla" - model = models.get_model(id=created_model.id) - assert model.description == "bla" - - @mock.patch("requests.sessions.Session.put") - def test_predict_on_model(self, mock_put): - mock_put.return_value = MockReturnValue(json_data={"data": {"predictions": [1, 2, 3]}}) - predictions = models.online_predict(model_id=1) - assert predictions == [1, 2, 3] - - @mock.patch("requests.sessions.Session.put") - def test_predict_on_model_prediction_error(self, mock_put): - mock_put.return_value = MockReturnValue(json_data={"error": {"message": "User error", "code": 200}}) - with pytest.raises(PredictionError, match="User error"): - models.online_predict(model_id=1) - - def test_deprecate_model(self, created_model): - res = models.deprecate_model(id=created_model.id) - assert isinstance(res, ModelResponse) - assert res.is_deprecated is True - model = models.get_model(id=created_model.id) - assert model.is_deprecated is True - - -class TestVersions: - model_version_response = { - "data": { - "items": [ - { - "isDeprecated": True, - "trainingDetails": { - "machineType": "string", - "scaleTier": "string", - "completedTime": 0, - "sourcePackageId": "string", - "args": "string", - }, - "name": "string", - "errorMsg": "string", - "modelId": 1, - "createdTime": 0, - "metadata": {"k": "v"}, - "id": 1, - "sourcePackageId": 1, - "status": "string", - "description": "string", - "project": "string", - } - ] - }, - "nextCursor": "string", - } - - @mock.patch("requests.sessions.Session.post") - def test_create_version(self, post_mock): - model_version_response = self.model_version_response.copy() - model_version_response["data"]["items"][0]["trainingDetails"] = None - post_mock.return_value = MockReturnValue(json_data=model_version_response) - res = models.create_model_version(model_id=1, name="mymodel", source_package_id=1) - assert isinstance(res, ModelVersionResponse) - assert 1 == res.id - - @mock.patch("requests.sessions.Session.post") - def test_deploy_version(self, post_mock): - model_version_response = self.model_version_response.copy() - model_version_response["data"]["items"][0]["trainingDetails"] = None - model_version_response["data"]["items"][0]["status"] = "DEPLOYING" - post_mock.return_value = MockReturnValue(json_data=model_version_response) - res = models.deploy_awaiting_model_version(model_id=1, version_id=1) - assert isinstance(res, ModelVersionResponse) - assert "DEPLOYING" == res.status - - @mock.patch("requests.sessions.Session.put") - @mock.patch("requests.sessions.Session.post") - def test_create_and_deploy_model_version(self, post_mock, put_mock): - model_version_response = self.model_version_response.copy() - model_version_response["data"]["items"][0]["trainingDetails"] = None - post_mock.side_effect = [ - MockReturnValue(json_data=model_version_response), - MockReturnValue(json_data={"data": {"uploadUrl": "https://upload.here"}}), - MockReturnValue(json_data={"data": {"uploadUrl": "https://upload.here"}}), - MockReturnValue(json_data=model_version_response), - ] - put_mock.return_value = MockReturnValue() - - artifacts_directory = os.path.join(os.path.dirname(__file__), "source_package_for_tests/artifacts") - model_version = models.deploy_model_version( - model_id=1, name="mymodel", source_package_id=1, artifacts_directory=artifacts_directory - ) - assert model_version.id == 1 - assert { - "description": "", - "metadata": {}, - "name": "mymodel", - "sourcePackageId": 1, - } == get_call_args_data_from_mock(post_mock, 0, decompress_gzip=True) - post_artifacts_call_args = [ - get_call_args_data_from_mock(post_mock, 1, decompress_gzip=True), - get_call_args_data_from_mock(post_mock, 2, decompress_gzip=True), - ] - assert {"name": "artifact1.txt"} in post_artifacts_call_args - assert {"name": "sub_dir/artifact2.txt"} in post_artifacts_call_args - assert {} == get_call_args_data_from_mock(post_mock, 3, decompress_gzip=True) - - @mock.patch("requests.sessions.Session.get") - def test_list_versions(self, get_mock): - get_mock.return_value = MockReturnValue(json_data=self.model_version_response) - res = models.list_model_versions(model_id=1) - assert len(res) > 0 - assert isinstance(res, ModelVersionCollectionResponse) - assert isinstance(res[:1], ModelVersionCollectionResponse) - assert isinstance(res[0], ModelVersionResponse) - for model in res: - assert isinstance(model, ModelVersionResponse) - assert res[0].to_json() == self.model_version_response["data"]["items"][0] - assert res[0].id == self.model_version_response["data"]["items"][0]["id"] - - @mock.patch("requests.sessions.Session.get") - def test_get_version(self, get_mock): - get_mock.return_value = MockReturnValue(json_data=self.model_version_response) - model_version = models.get_model_version(model_id=1, version_id=1) - assert isinstance(model_version, ModelVersionResponse) - assert model_version.id == self.model_version_response["data"]["items"][0]["id"] - - @mock.patch("requests.sessions.Session.post") - def test_train_and_deploy_version(self, post_mock): - post_mock.return_value = MockReturnValue(json_data=self.model_version_response) - res = models.train_and_deploy_model_version(model_id=1, name="mymodel", source_package_id=1) - assert isinstance(res, ModelVersionResponse) - - @mock.patch("requests.sessions.Session.post") - def test_train_and_deploy_version_data_spec_arg(self, post_mock, mock_data_spec): - post_mock.return_value = MockReturnValue(json_data=self.model_version_response) - models.train_and_deploy_model_version( - model_id=1, name="mymodel", source_package_id=1, args={"data_spec": mock_data_spec} - ) - data_sent_to_api = json.loads(gzip.decompress(post_mock.call_args[1]["data"]).decode()) - assert {"spec": "spec"} == data_sent_to_api["trainingDetails"]["args"]["data_spec"] - - @mock.patch("requests.sessions.Session.delete") - def test_delete_version(self, delete_mock): - delete_mock.return_value = MockReturnValue() - res = models.delete_model_version(model_id=1, version_id=1) - assert res is None - - @mock.patch("requests.sessions.Session.get") - def test_list_artifacts(self, get_mock): - get_mock.return_value = MockReturnValue(json_data={"data": {"items": [{"name": "a1", "size": 1}]}}) - res = models.list_artifacts(model_id=1, version_id=1) - assert len(res) > 0 - assert isinstance(res, ModelArtifactCollectionResponse) - assert isinstance(res[:1], ModelArtifactCollectionResponse) - assert isinstance(res[0], ModelArtifactResponse) - assert res[0].name == "a1" - assert res[0].size == 1 - - @mock.patch("requests.sessions.Session.get") - def test_download_artifact(self, mock_get): - mock_get.side_effect = [ - MockReturnValue(json_data={"data": {"downloadUrl": "https://download.me"}}), - MockReturnValue(content=b"content"), - ] - models.download_artifact(model_id=1, version_id=1, name="a1", directory=os.getcwd()) - file_path = os.path.join(os.getcwd(), "a1") - assert os.path.isfile(file_path) - with open(file_path, "rb") as f: - assert b"content" == f.read() - os.remove(file_path) - - @pytest.fixture(scope="class") - def artifact_file_path(self): - file_path = "/tmp/my_artifact.txt" - with open(file_path, "w") as f: - f.write("content") - yield file_path - os.remove(file_path) - - @mock.patch("requests.sessions.Session.put") - @mock.patch("requests.sessions.Session.post") - def test_upload_artifact_from_file(self, post_mock, put_mock, artifact_file_path): - post_mock.return_value = MockReturnValue(json_data={"data": {"uploadUrl": "https://upload.here"}}) - put_mock.return_value = MockReturnValue() - models.upload_artifact_from_file(model_id=1, version_id=1, name="my_artifact.txt", file_path=artifact_file_path) - - @mock.patch("requests.sessions.Session.put") - @mock.patch("requests.sessions.Session.post") - def test_upload_artifacts_from_directory(self, post_mock, put_mock): - artifacts_directory = os.path.join(os.path.dirname(__file__), "source_package_for_tests/artifacts") - post_mock.side_effect = [ - MockReturnValue(json_data={"data": {"uploadUrl": "https://upload.here"}}), - MockReturnValue(json_data={"data": {"uploadUrl": "https://upload.here"}}), - ] - put_mock.return_value = MockReturnValue() - - models.upload_artifacts_from_directory(model_id=1, version_id=1, directory=artifacts_directory) - - post_artifacts_call_args = [ - get_call_args_data_from_mock(post_mock, 0, decompress_gzip=True), - get_call_args_data_from_mock(post_mock, 1, decompress_gzip=True), - ] - assert {"name": "artifact1.txt"} in post_artifacts_call_args - assert {"name": "sub_dir/artifact2.txt"} in post_artifacts_call_args - - @mock.patch("requests.sessions.Session.put") - def test_deprecate_model_version(self, mock_put): - mock_put.return_value = MockReturnValue(json_data=self.model_version_response) - res = models.deprecate_model_version(model_id=1, version_id=1) - assert isinstance(res, ModelVersionResponse) - assert res.is_deprecated is True - - @mock.patch("requests.sessions.Session.put") - def test_update_model_version(self, mock_put): - updated_model_version = self.model_version_response.copy() - updated_model_version["data"]["items"][0]["description"] = "blabla" - mock_put.return_value = MockReturnValue(json_data=updated_model_version) - res = models.update_model_version(model_id=1, version_id=1, description="blabla") - assert isinstance(res, ModelVersionResponse) - assert res.description == "blabla" - - @mock.patch("requests.sessions.Session.get") - def test_get_model_version_log(self, mock_get): - mock_get.return_value = MockReturnValue( - json_data={"data": {"predict": ["l1", "l2", "l3"], "train": ["l1", "l2", "l3"]}} - ) - res = models.get_logs(model_id=1, version_id=1, log_type="both") - assert isinstance(res, ModelLogResponse) - assert res.prediction_logs == ["l1", "l2", "l3"] - assert res.training_logs == ["l1", "l2", "l3"] - assert ( - res.__str__() == '{\n "predict": [\n "l1",\n "l2",\n "l3"\n ],\n ' - '"train": [\n "l1",\n "l2",\n "l3"\n ]\n}' - ) - - @mock.patch("requests.sessions.Session.put") - def test_predict_on_model_version(self, mock_put): - mock_put.return_value = MockReturnValue(json_data={"data": {"predictions": [1, 2, 3]}}) - predictions = models.online_predict(model_id=1, version_id=1) - assert predictions == [1, 2, 3] - - @mock.patch("requests.sessions.Session.put") - def test_predict_instance_is_data_spec(self, mock_put, mock_data_spec): - mock_put.return_value = MockReturnValue(json_data={"data": {"predictions": [1, 2, 3]}}) - models.online_predict(model_id=1, version_id=1, instances=[mock_data_spec, mock_data_spec]) - data_sent_to_api = json.loads(gzip.decompress(mock_put.call_args[1]["data"]).decode()) - for instance in data_sent_to_api["instances"]: - assert {"spec": "spec"} == instance - - @mock.patch("requests.sessions.Session.put") - def test_predict_on_model_version_prediction_error(self, mock_put): - mock_put.return_value = MockReturnValue(json_data={"error": {"message": "User error", "code": 200}}) - with pytest.raises(PredictionError, match="User error"): - models.online_predict(model_id=1, version_id=1) - - -class TestSchedules: - schedule_response = { - "data": { - "items": [ - { - "isDeprecated": False, - "name": "test-schedule", - "dataSpec": {"spec": "spec"}, - "modelId": 123, - "createdTime": 0, - "metadata": {"k": "v"}, - "id": 123, - "args": {"k": "v"}, - "description": "string", - } - ] - }, - "nextCursor": "string", - } - - @mock.patch("requests.sessions.Session.post") - def test_create_schedule(self, mock_post): - mock_post.return_value = MockReturnValue(json_data=self.schedule_response) - res = schedules.create_schedule( - model_id=123, name="myschedule", schedule_data_spec={"spec": "spec"}, args={"k": "v"}, metadata={"k": "v"} - ) - assert isinstance(res, ScheduleResponse) - assert res.id == 123 - - @mock.patch("requests.sessions.Session.post") - def test_create_schedule_with_data_spec_objects(self, mock_post, mock_data_spec): - mock_post.return_value = MockReturnValue(json_data=self.schedule_response) - res = schedules.create_schedule( - model_id=123, name="myschedule", schedule_data_spec=mock_data_spec, args={"k": "v"}, metadata={"k": "v"} - ) - assert isinstance(res, ScheduleResponse) - assert res.id == 123 - - data_sent_to_api = json.loads(gzip.decompress(mock_post.call_args[1]["data"]).decode()) - actual_data_spec = data_sent_to_api["dataSpec"] - - assert {"spec": "spec"} == actual_data_spec - - @mock.patch("requests.sessions.Session.get") - def test_list_schedules(self, mock_get): - mock_get.return_value = MockReturnValue(json_data=self.schedule_response) - res = schedules.list_schedules(limit=1) - assert len(res) > 0 - assert isinstance(res, ScheduleCollectionResponse) - assert isinstance(res[:1], ScheduleCollectionResponse) - assert isinstance(res[0], ScheduleResponse) - assert self.schedule_response["data"]["items"][0]["name"] == res[0].name - - @mock.patch("requests.sessions.Session.get") - def test_get_schedule(self, mock_get): - mock_get.return_value = MockReturnValue(json_data=self.schedule_response) - res = schedules.get_schedule(id=1) - assert isinstance(res, ScheduleResponse) - assert self.schedule_response["data"]["items"][0]["name"] == res.name - - @mock.patch("requests.sessions.Session.put") - def test_deprecate_schedule(self, mock_put): - depr_schedule_response = self.schedule_response.copy() - depr_schedule_response["data"]["items"][0]["isDeprecated"] = True - mock_put.return_value = MockReturnValue(json_data=depr_schedule_response) - - res = schedules.deprecate_schedule(id=1) - assert res.is_deprecated is True - - @mock.patch("requests.sessions.Session.delete") - def test_delete_schedule(self, mock_delete): - mock_delete.return_value = MockReturnValue() - res = schedules.delete_schedule(id=1) - assert res is None diff --git a/tests/test_experimental/test_sequences.py b/tests/test_experimental/test_sequences.py deleted file mode 100644 index 4c50a731f3..0000000000 --- a/tests/test_experimental/test_sequences.py +++ /dev/null @@ -1,126 +0,0 @@ -from time import sleep -from typing import List - -import pandas as pd -import pytest - -from cognite.client import APIError, CogniteClient -from cognite.client.experimental.sequences import Column, Row, RowValue, Sequence, SequenceDataResponse -from tests.conftest import generate_random_string - -sequences = CogniteClient().experimental.sequences - -# This variable will hold the ID of the sequence that is created in one of the test fixtures of this class. -CREATED_SEQUENCE_ID = None -# This variable holds the external id used for the sequence that'll be created (and deleted) in these tests -SEQUENCE_EXTERNAL_ID = "external_id" + generate_random_string(10) - - -class TestSequences: - @pytest.fixture(scope="class") - def sequence_that_isnt_created(self): - """Returns a Sequence that hasn't been created yet. (It does not have an ID)""" - global SEQUENCE_EXTERNAL_ID - - return Sequence( - id=None, - name="test_sequence", - external_id=SEQUENCE_EXTERNAL_ID, - asset_id=None, - columns=[ - Column(id=None, name="test_column", external_id="external_id", value_type="STRING", metadata={}), - Column(id=None, name="test_column2", external_id="external_id2", value_type="STRING", metadata={}), - ], - description="Test sequence", - metadata={}, - ) - - @pytest.fixture(scope="class") - def sequence_that_is_created_retrieved_by_id(self, sequence_that_isnt_created): - """Returns the created sequence by using the cognite id""" - global CREATED_SEQUENCE_ID - if CREATED_SEQUENCE_ID: - res = sequences.get_sequence_by_id(CREATED_SEQUENCE_ID) - return res - created_sequence = sequences.post_sequences([sequence_that_isnt_created]) - CREATED_SEQUENCE_ID = created_sequence.id - return created_sequence - - @pytest.fixture(scope="class") - def sequence_that_is_created_retrieved_by_external_id(self, sequence_that_isnt_created): - """Returns the created sequence by using the external id""" - global CREATED_SEQUENCE_ID, SEQUENCE_EXTERNAL_ID - if CREATED_SEQUENCE_ID: - res = sequences.list_sequences(SEQUENCE_EXTERNAL_ID) - while len(res) == 0: - res = sequences.list_sequences(SEQUENCE_EXTERNAL_ID) - sleep(0.5) - res = sequences.get_sequence_by_external_id(SEQUENCE_EXTERNAL_ID) - return res - created_sequence = sequences.post_sequences([sequence_that_isnt_created]) - CREATED_SEQUENCE_ID = created_sequence.id - return created_sequence - - def test_get_sequence_by_id(self, sequence_that_is_created_retrieved_by_id, sequence_that_isnt_created): - global CREATED_SEQUENCE_ID - assert isinstance(sequence_that_is_created_retrieved_by_id, Sequence) - assert sequence_that_is_created_retrieved_by_id.id == CREATED_SEQUENCE_ID - assert sequence_that_is_created_retrieved_by_id.name == sequence_that_isnt_created.name - - def test_get_sequence_by_external_id( - self, sequence_that_is_created_retrieved_by_external_id, sequence_that_isnt_created - ): - global CREATED_SEQUENCE_ID - assert isinstance(sequence_that_is_created_retrieved_by_external_id, Sequence) - assert sequence_that_is_created_retrieved_by_external_id.id == CREATED_SEQUENCE_ID - assert sequence_that_is_created_retrieved_by_external_id.name == sequence_that_isnt_created.name - - @pytest.mark.skip - def test_post_data_to_sequence_and_get_data_from_sequence(self, sequence_that_is_created_retrieved_by_id): - # Prepare some data to post - rows = [ - Row( - row_number=1, - values=[ - RowValue(column_id=sequence_that_is_created_retrieved_by_id.columns[0].id, value="42"), - RowValue(column_id=sequence_that_is_created_retrieved_by_id.columns[1].id, value="43"), - ], - ) - ] - # Post data - res = sequences.post_data_to_sequence(id=sequence_that_is_created_retrieved_by_id.id, rows=rows) - assert res == {} - # Sleep a little, to give the api a chance to process the data - import time - - time.sleep(5) - # Get the data - sequenceDataResponse = sequences.get_data_from_sequence( - id=sequence_that_is_created_retrieved_by_id.id, - inclusive_from=1, - inclusive_to=1, - limit=1, - column_ids=[ - sequence_that_is_created_retrieved_by_id.columns[0].id, - sequence_that_is_created_retrieved_by_id.columns[1].id, - ], - ) - # Verify that the data is the same - assert rows[0].rowNumber == sequenceDataResponse.rows[0].rowNumber - assert rows[0].values[0].columnId == sequenceDataResponse.rows[0].values[0].columnId - assert rows[0].values[0].value == sequenceDataResponse.rows[0].values[0].value - assert rows[0].values[1].columnId == sequenceDataResponse.rows[0].values[1].columnId - assert rows[0].values[1].value == sequenceDataResponse.rows[0].values[1].value - - # Verify that we can get the data as a pandas dataframe - dataframe = sequenceDataResponse.to_pandas() - assert isinstance(dataframe, pd.DataFrame) - assert len(rows[0].values) == len(dataframe.values[0]) - assert rows[0].values[0].value == dataframe.values[0][0] - assert rows[0].values[1].value == dataframe.values[0][1] - - def test_delete_sequence_by_id(self, sequence_that_is_created_retrieved_by_id): - sequences.delete_sequence_by_id(sequence_that_is_created_retrieved_by_id.id) - # Check that we now can't fetch it - with pytest.raises(APIError): - sequences.get_sequence_by_id(sequence_that_is_created_retrieved_by_id.id) diff --git a/tests/test_experimental/test_time_series.py b/tests/test_experimental/test_time_series.py deleted file mode 100644 index 80c532c301..0000000000 --- a/tests/test_experimental/test_time_series.py +++ /dev/null @@ -1,46 +0,0 @@ -from random import randint -from time import sleep - -import pytest - -from cognite.client import CogniteClient -from cognite.client.experimental.time_series import TimeSeriesResponse -from cognite.client.stable.time_series import TimeSeries -from tests.conftest import TEST_TS_1_NAME - -stable_time_series = CogniteClient().time_series -time_series = CogniteClient().experimental.time_series - - -@pytest.fixture -def new_ts_id(): - name = "test_ts_{}".format(randint(1, 2 ** 53 - 1)) - stable_time_series.post_time_series([TimeSeries(name)]) - - res = stable_time_series.get_time_series(prefix=name) - while len(res) == 0: - res = stable_time_series.get_time_series(prefix=name) - sleep(0.5) - yield res[0].id - - -class TestTimeseries: - def test_delete_time_series_by_id(self, new_ts_id): - res = time_series.delete_time_series_by_id([new_ts_id]) - assert res is None - - @pytest.fixture(scope="class") - def get_time_series_by_id_response_obj(self, time_series_in_cdp): - yield time_series.get_time_series_by_id(id=time_series_in_cdp[0]) - - @pytest.fixture(scope="class") - def get_multiple_time_series_by_id_response_obj(self, time_series_in_cdp): - yield time_series.get_multiple_time_series_by_id(ids=[time_series_in_cdp[0]]) - - def test_get_time_series_by_id(self, get_time_series_by_id_response_obj): - assert isinstance(get_time_series_by_id_response_obj, TimeSeriesResponse) - assert get_time_series_by_id_response_obj.to_json()[0]["name"] == TEST_TS_1_NAME - - def test_get_multiple_time_series_by_id(self, get_multiple_time_series_by_id_response_obj): - assert isinstance(get_multiple_time_series_by_id_response_obj, TimeSeriesResponse) - assert get_multiple_time_series_by_id_response_obj.to_json()[0]["name"] == TEST_TS_1_NAME diff --git a/tests/test_stable/__init__.py b/tests/test_stable/__init__.py deleted file mode 100644 index 56fafa58b3..0000000000 --- a/tests/test_stable/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- diff --git a/tests/test_stable/test_assets.py b/tests/test_stable/test_assets.py deleted file mode 100644 index 3361136bfe..0000000000 --- a/tests/test_stable/test_assets.py +++ /dev/null @@ -1,104 +0,0 @@ -import time - -import pandas as pd -import pytest - -from cognite.client import CogniteClient -from cognite.client.stable.assets import Asset, AssetListResponse, AssetResponse -from tests.conftest import generate_random_string - -assets = CogniteClient(debug=True).assets - - -@pytest.fixture(scope="module") -def get_asset_subtree_response(): - return assets.get_asset_subtree(asset_id=6354653755843357, limit=2) - - -@pytest.fixture(scope="module") -def get_assets_response(): - return assets.get_assets(limit=1) - - -def test_get_assets_response_object(get_assets_response): - assert isinstance(get_assets_response, AssetListResponse) - assert get_assets_response.next_cursor() is not None - assert get_assets_response.previous_cursor() is None - assert len(get_assets_response) - assert isinstance(get_assets_response[0], AssetResponse) - assert isinstance(get_assets_response[:1], AssetListResponse) - assert len(get_assets_response[:1]) == 1 - - -def test_get_assets_with_metadata_args(): - res = assets.get_assets(limit=1, metadata={"something": "something"}) - assert not res.to_json() - - -def test_get_asset(): - res = assets.get_asset(6354653755843357) - assert isinstance(res, AssetResponse) - assert isinstance(res.to_json(), dict) - assert isinstance(res.to_pandas(), pd.DataFrame) - assert res.to_pandas().shape[1] == 1 - - -def test_attributes_not_none(): - asset = assets.get_asset(6354653755843357) - for key, val in asset.__dict__.items(): - if key is "metadata" or (key is "parent_id" and asset.depth == 0): - assert val is None - else: - assert val is not None, "{} is None".format(key) - - -def test_asset_subtree_object(get_asset_subtree_response): - assert isinstance(get_asset_subtree_response, AssetListResponse) - assert get_asset_subtree_response.next_cursor() is not None - assert get_asset_subtree_response.previous_cursor() is None - - -def test_json(get_asset_subtree_response): - assert isinstance(get_asset_subtree_response.to_json(), list) - - -def test_pandas(get_asset_subtree_response): - assert isinstance(get_asset_subtree_response.to_pandas(), pd.DataFrame) - - -@pytest.fixture(scope="module") -def created_asset(): - random_asset_name = "test_asset" + generate_random_string(10) - a1 = Asset(name=random_asset_name) - res = assets.post_assets([a1]) - assert isinstance(res, AssetListResponse) - assert res.to_json()[0]["name"] == random_asset_name - assert res.to_json()[0].get("id") != None - - assets_response = assets.get_assets(random_asset_name, depth=0) - while len(assets_response) == 0: - assets_response = assets.get_assets(random_asset_name, depth=0) - time.sleep(0.5) - return assets_response[0] - - -def test_update_asset(created_asset): - new_name = generate_random_string(10) - res = assets.update_asset(created_asset.id, name=new_name) - assert new_name == res.name - - -def test_update_multiple_assets(created_asset): - new_name = generate_random_string(10) - res = assets.update_assets([Asset(id=created_asset.id, name=new_name)]) - assert new_name == res[0].name - - -def test_delete_assets(created_asset): - res = assets.delete_assets([created_asset.id]) - assert res is None - - -def test_search_for_assets(): - res = assets.search_for_assets() - assert len(res.to_json()) > 0 diff --git a/tests/test_stable/test_datapoints.py b/tests/test_stable/test_datapoints.py deleted file mode 100644 index 225e631992..0000000000 --- a/tests/test_stable/test_datapoints.py +++ /dev/null @@ -1,257 +0,0 @@ -from copy import copy -from random import randint -from unittest import mock - -import numpy as np -import pandas as pd -import pytest - -from cognite.client import CogniteClient -from cognite.client._api_client import APIClient -from cognite.client.stable.datapoints import ( - Datapoint, - DatapointsQuery, - DatapointsResponse, - LatestDatapointResponse, - TimeseriesWithDatapoints, -) -from cognite.client.stable.time_series import TimeSeries -from tests.conftest import ( - TEST_TS_1_NAME, - TEST_TS_2_NAME, - TEST_TS_REASONABLE_INTERVAL, - TEST_TS_REASONABLE_INTERVAL_DATETIME, -) - -client = CogniteClient() - -TS_NAME = None - - -@pytest.fixture(autouse=True, scope="class") -def ts_name(): - global TS_NAME - TS_NAME = "test_ts_{}".format(randint(1, 2 ** 53 - 1)) - - -@pytest.fixture(scope="class") -def datapoints_fixture(): - tso = TimeSeries(TS_NAME) - client.time_series.post_time_series([tso]) - yield - client.time_series.delete_time_series(TS_NAME) - - -dps_params = [ - TEST_TS_REASONABLE_INTERVAL, - TEST_TS_REASONABLE_INTERVAL_DATETIME, - { - "start": TEST_TS_REASONABLE_INTERVAL_DATETIME["start"], - "end": TEST_TS_REASONABLE_INTERVAL_DATETIME["end"], - "protobuf": True, - }, -] - - -@pytest.mark.usefixtures("datapoints_fixture") -class TestDatapoints: - @pytest.fixture(scope="class", params=dps_params) - def get_dps_response_obj(self, request): - res = client.datapoints.get_datapoints( - name=TEST_TS_1_NAME, - start=request.param["start"], - end=request.param["end"], - include_outside_points=True, - protobuf=request.param.get("protobuf", False), - ) - yield res - - @pytest.fixture(scope="class") - def get_dps_aggregates_response_obj(self): - res = client.datapoints.get_datapoints( - name=TEST_TS_1_NAME, - start=TEST_TS_REASONABLE_INTERVAL["start"], - end=TEST_TS_REASONABLE_INTERVAL["end"], - aggregates=["min", "max"], - granularity="1s", - ) - yield res - - def test_post_datapoints(self): - dps = [Datapoint(i, i * 100) for i in range(10)] - res = client.datapoints.post_datapoints(TS_NAME, datapoints=dps) - assert res is None - - def test_post_datapoints_float_timestamp(self): - dps = [Datapoint(float(i), i * 100) for i in range(10)] - res = client.datapoints.post_datapoints(TS_NAME, datapoints=dps) - assert res is None - - def test_post_datapoints_frame(self): - data = pd.DataFrame() - data["timestamp"] = [int(1537208777557 + 1000 * i) for i in range(0, 100)] - X = data["timestamp"].values.astype(float) - data["X"] = X ** 2 - data["Y"] = 1.0 / (1 + X) - - for name in data.drop(["timestamp"], axis=1).columns: - ts = TimeSeries(name=name, description="To be deleted") - try: - client.time_series.post_time_series([ts]) - except: - pass - - res = client.datapoints.post_datapoints_frame(data) - assert res is None - - def test_get_datapoints(self, get_dps_response_obj): - assert isinstance(get_dps_response_obj, DatapointsResponse) - - def test_get_dps_output_formats(self, get_dps_response_obj): - assert isinstance(get_dps_response_obj.to_pandas(), pd.DataFrame) - assert isinstance(get_dps_response_obj.to_json(), dict) - - def test_attributes_not_none(self, get_dps_response_obj): - for key, val in get_dps_response_obj.__dict__.items(): - assert val is not None - - def test_get_dps_correctly_spaced(self, get_dps_response_obj): - timestamps = get_dps_response_obj.to_pandas().timestamp.values - deltas = np.diff(timestamps, 1) - assert (deltas != 0).all() - assert (deltas % 10000 == 0).all() - - def test_get_dps_with_limit(self): - res = client.datapoints.get_datapoints(name=TEST_TS_1_NAME, start=0, limit=1) - assert len(res.to_json().get("datapoints")) == 1 - - def test_get_dps_with_end_now(self): - res = client.datapoints.get_datapoints(name=TEST_TS_1_NAME, start=0, end="now", limit=100) - assert len(res.to_json().get("datapoints")) == 100 - - def test_get_dps_with_aggregates(self, get_dps_aggregates_response_obj): - dps = get_dps_aggregates_response_obj.to_json()["datapoints"] - assert len(dps[0].keys()) == 3, "Datapoints should have 3 columns: timestamp, min, max" - - -class TestLatest: - def test_get_latest(self): - response = client.datapoints.get_latest(TEST_TS_1_NAME) - assert set(list(response.to_json().keys())) == {"timestamp", "value"} - assert isinstance(response, LatestDatapointResponse) - assert isinstance(response.to_pandas(), pd.DataFrame) - assert isinstance(response.to_json(), dict) - timestamp = response.to_json()["timestamp"] - response = client.datapoints.get_latest(TEST_TS_1_NAME, before=timestamp) - assert response.to_json()["timestamp"] < timestamp - - -class TestDatapointsFrame: - @pytest.fixture(scope="class", params=dps_params[:2]) - def get_datapoints_frame_response_obj(self, request): - yield client.datapoints.get_datapoints_frame( - time_series=[TEST_TS_1_NAME], - start=request.param["start"], - end=request.param["end"], - aggregates=["avg"], - granularity="1m", - ) - - def test_get_dps_frame_output_format(self, get_datapoints_frame_response_obj): - assert isinstance(get_datapoints_frame_response_obj, pd.DataFrame) - - def test_get_dps_frame_correctly_spaced(self, get_datapoints_frame_response_obj): - timestamps = get_datapoints_frame_response_obj.timestamp.values - deltas = np.diff(timestamps, 1) - assert (deltas != 0).all() - assert (deltas % 60000 == 0).all() - - def test_get_dps_frame_with_limit(self): - df = client.datapoints.get_datapoints_frame( - time_series=[TEST_TS_1_NAME], aggregates=["avg"], granularity="1m", start=0, limit=1 - ) - assert df.shape[0] == 1 - - -class TestMultiTimeseriesDatapoints: - @pytest.fixture(scope="class", params=dps_params[:1]) - def get_multi_time_series_dps_response_obj(self, request): - dq1 = DatapointsQuery(TEST_TS_1_NAME) - dq2 = DatapointsQuery(TEST_TS_2_NAME, aggregates=["avg"], granularity="30s") - dq1_copy = copy(dq1) - dq2_copy = copy(dq2) - yield list( - client.datapoints.get_multi_time_series_datapoints( - datapoints_queries=[dq1, dq2], - start=request.param["start"], - end=request.param["end"], - aggregates=["avg"], - granularity="60s", - ) - ) - for val, val_c in zip(dq1.__dict__.values(), dq1_copy.__dict__.values()): - assert val == val_c - - for val, val_c in zip(dq2.__dict__.values(), dq2_copy.__dict__.values()): - assert val == val_c - - def test_post_multitag_datapoints(self): - timeseries_with_too_many_datapoints = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(100001)] - ) - timeseries_with_99999_datapoints = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(99999)] - ) - - with mock.patch.object(APIClient, "_post") as post_request_mock: - post_request_mock = post_request_mock - - client.datapoints.post_multi_time_series_datapoints([timeseries_with_too_many_datapoints]) - assert post_request_mock.call_count == 2 - - with mock.patch.object(APIClient, "_post") as post_request_mock: - post_request_mock = post_request_mock - - client.datapoints.post_multi_time_series_datapoints( - [timeseries_with_99999_datapoints, timeseries_with_too_many_datapoints] - ) - assert post_request_mock.call_count == 2 - - def test_get_multi_time_series_dps_output_format(self, get_multi_time_series_dps_response_obj): - assert isinstance(get_multi_time_series_dps_response_obj, list) - assert isinstance(get_multi_time_series_dps_response_obj[0], DatapointsResponse) - for dpr in get_multi_time_series_dps_response_obj: - assert isinstance(dpr, DatapointsResponse) - - def test_get_multi_time_series_dps_response_length(self, get_multi_time_series_dps_response_obj): - assert len(list(get_multi_time_series_dps_response_obj)) == 2 - - def test_get_multi_timeseries_dps_correctly_spaced(self, get_multi_time_series_dps_response_obj): - m = list(get_multi_time_series_dps_response_obj) - timestamps = m[0].to_pandas().timestamp.values - deltas = np.diff(timestamps, 1) - assert (deltas != 0).all() - assert (deltas % 60000 == 0).all() - timestamps = m[1].to_pandas().timestamp.values - deltas = np.diff(timestamps, 1) - assert (deltas != 0).all() - assert (deltas % 30000 == 0).all() - - def test_split_TimeseriesWithDatapoints_if_over_limit(self): - timeseries_with_datapoints_over_limit = TimeseriesWithDatapoints( - name="test", datapoints=[Datapoint(x, x) for x in range(1000)] - ) - - result = client.datapoints._split_TimeseriesWithDatapoints_if_over_limit( - timeseries_with_datapoints_over_limit, 100 - ) - - assert isinstance(result[0], TimeseriesWithDatapoints) - assert len(result) == 10 - - result = client.datapoints._split_TimeseriesWithDatapoints_if_over_limit( - timeseries_with_datapoints_over_limit, 1000 - ) - - assert isinstance(result[0], TimeseriesWithDatapoints) - assert len(result) == 1 diff --git a/tests/test_stable/test_dto.py b/tests/test_stable/test_dto.py deleted file mode 100644 index 49f37d7c80..0000000000 --- a/tests/test_stable/test_dto.py +++ /dev/null @@ -1,35 +0,0 @@ -from copy import deepcopy - -import pytest - -from cognite.client.stable.events import EventListResponse, EventResponse -from cognite.client.stable.files import FileInfoResponse -from cognite.client.stable.time_series import TimeSeriesResponse - - -@pytest.fixture(scope="module", params=["ts", "file", "event", "eventlist"]) -def get_response_obj(request): - TS_INTERNAL_REPR = {"data": {"items": [{"name": "0", "metadata": {"md1": "val1"}}]}} - EVENT_LIST_INTERNAL_REPR = {"data": {"items": [{"id": 0, "metadata": {"md1": "val1"}}]}} - EVENT_INTERNAL_REPR = {"data": {"items": [{"id": 0, "metadata": {"md1": "val1"}, "assetIds": []}]}} - FILE_INFO_INTERNAL_REPR = {"data": {"items": [{"id": 0, "metadata": {"md1": "val1"}}]}} - - response = None - if request.param == "ts": - response = TimeSeriesResponse(TS_INTERNAL_REPR) - elif request.param == "file": - response = FileInfoResponse(FILE_INFO_INTERNAL_REPR) - elif request.param == "eventlist": - response = EventListResponse(EVENT_LIST_INTERNAL_REPR) - elif request.param == "event": - response = EventResponse(EVENT_INTERNAL_REPR) - - yield response - - -class TestDTOs: - def test_internal_representation_not_mutated(self, get_response_obj): - repr = deepcopy(get_response_obj.internal_representation) - get_response_obj.to_pandas() - get_response_obj.to_json() - assert repr == get_response_obj.internal_representation diff --git a/tests/test_stable/test_events.py b/tests/test_stable/test_events.py deleted file mode 100644 index fe82a55e9c..0000000000 --- a/tests/test_stable/test_events.py +++ /dev/null @@ -1,86 +0,0 @@ -import pandas as pd -import pytest - -import cognite.client.stable.events -from cognite.client import APIError, CogniteClient - -events = CogniteClient().events - - -@pytest.fixture(scope="module") -def get_post_event_obj(): - event = cognite.client.stable.events.Event(start_time=1521500400000, end_time=1521586800000, description="hahaha") - res = events.post_events([event]) - yield res - ids = list(ev["id"] for ev in res.to_json()) - events.delete_events(ids) - - -def test_post_events(get_post_event_obj): - assert isinstance(get_post_event_obj, cognite.client.stable.events.EventListResponse) - assert isinstance(get_post_event_obj.to_pandas(), pd.DataFrame) - assert isinstance(get_post_event_obj.to_json(), list) - - -def test_attributes_not_none(get_post_event_obj): - assert isinstance(get_post_event_obj, cognite.client.stable.events.EventListResponse) - assert isinstance(get_post_event_obj.to_pandas(), pd.DataFrame) - assert isinstance(get_post_event_obj.to_json(), list) - - -def test_post_events_length(get_post_event_obj): - assert len(get_post_event_obj.to_json()) == 1 - - -def test_get_event(get_post_event_obj): - id = get_post_event_obj.to_json()[0]["id"] - res = events.get_event(event_id=id) - assert isinstance(res, cognite.client.stable.events.EventResponse) - assert isinstance(res.to_pandas(), pd.DataFrame) - assert isinstance(res.to_json(), dict) - assert res.to_pandas().shape[1] == 1 - - -def test_get_event_invalid_id(): - with pytest.raises(APIError): - events.get_event(123456789) - - -def test_get_events(): - res = events.get_events(min_start_time=1521500399999, max_start_time=1521500400001) - assert isinstance(res, cognite.client.stable.events.EventListResponse) - assert isinstance(res.to_pandas(), pd.DataFrame) - assert isinstance(res.to_json(), list) - assert isinstance(res[0], cognite.client.stable.events.EventResponse) - assert isinstance(res[:1], cognite.client.stable.events.EventListResponse) - assert len(res[:1]) == 1 - for event in res: - assert isinstance(event, cognite.client.stable.events.EventResponse) - - -def test_get_events_invalid_param_combination(): - with pytest.raises(APIError, match="disabled"): - events.get_events(type="bla", asset_id=1) - - -def test_get_events_empty(): - res = events.get_events(asset_id=0) - assert res.to_pandas().empty - assert len(res.to_json()) == 0 - - -@pytest.fixture() -def post_event(): - event = cognite.client.stable.events.Event(start_time=1521500400000, end_time=1521586800000) - res = events.post_events([event]) - return res - - -def test_delete_event(post_event): - id = post_event.to_json()[0]["id"] - res = events.delete_events([id]) - assert res is None - - -def test_search_for_events(get_post_event_obj): - events.search_for_events(description="hahaha") diff --git a/tests/test_stable/test_files.py b/tests/test_stable/test_files.py deleted file mode 100644 index b685e71c38..0000000000 --- a/tests/test_stable/test_files.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -from time import sleep - -import pandas as pd -import pytest - -from cognite.client import CogniteClient -from cognite.client.stable.files import FileInfoResponse, FileListResponse - -files = CogniteClient().files - - -def test_upload_file_metadata(): - response = files.upload_file("test_file", source="sdk-tests", overwrite=True) - assert response.get("uploadURL") is not None - assert response.get("fileId") is not None - - -def test_upload_file(tmpdir): - file_path = os.path.join(str(tmpdir), "test_file.txt") - tmpdir.join("test_file.txt").write("This is a test file.") - with pytest.warns(UserWarning): - response = files.upload_file("test_file", file_path, source="sdk-tests", overwrite=True) - assert response.get("uploadURL") is None - assert response.get("fileId") is not None - - -def test_list_files(): - response = files.list_files(limit=3) - assert isinstance(response, FileListResponse) - assert isinstance(response.to_pandas(), pd.DataFrame) - assert isinstance(response.to_json(), list) - assert len(response.to_json()) > 0 and len(response.to_json()) <= 3 - assert isinstance(response[0], FileInfoResponse) - assert isinstance(response[:1], FileListResponse) - assert len(response[:1]) == 1 - - -def test_list_files_empty(): - response = files.list_files(source="not_a_source") - assert response.to_pandas().empty - assert len(response.to_json()) == 0 - - -@pytest.fixture(scope="module") -def file_id(): - res = files.list_files(name="test_file", source="sdk-tests", limit=1) - while len(res) == 0: - res = files.list_files(name="test_file", source="sdk-tests", limit=1) - sleep(0.5) - return res.to_json()[0]["id"] - - -def test_get_file_info(file_id): - response = files.get_file_info(file_id) - assert isinstance(response, FileInfoResponse) - assert isinstance(response.to_json(), dict) - assert isinstance(response.to_pandas(), pd.DataFrame) - assert response.id == file_id - - -@pytest.mark.parametrize("get_contents", [True, False]) -def test_download_files(file_id, get_contents): - try: - response = files.download_file(file_id, get_contents) - if get_contents: - assert isinstance(response, bytes) - else: - assert isinstance(response, str) - except Exception as e: - print("Failed to download file: ", e) - - -def test_delete_file(file_id): - response = files.delete_files([file_id]) - assert file_id in response["deleted"] or file_id in response["failed"] diff --git a/tests/test_stable/test_raw.py b/tests/test_stable/test_raw.py deleted file mode 100644 index 90a355562f..0000000000 --- a/tests/test_stable/test_raw.py +++ /dev/null @@ -1,124 +0,0 @@ -from random import randint - -import numpy as np -import pandas as pd -import pytest - -from cognite.client import APIError, CogniteClient -from cognite.client.stable.raw import RawResponse, RawRow - -raw = CogniteClient().raw - -DB_NAME = None -TABLE_NAME = None -ROW_KEY = None -ROW_COLUMNS = None - - -@pytest.fixture(autouse=True, scope="class") -def db_name(): - global DB_NAME - DB_NAME = "test_db_{}".format(randint(1, 2 ** 53 - 1)) - - -@pytest.fixture(autouse=True, scope="class") -def table_name(): - global TABLE_NAME - TABLE_NAME = "test_table_{}".format(randint(1, 2 ** 53 - 1)) - - -@pytest.fixture(autouse=True, scope="class") -def row_key(): - global ROW_KEY - ROW_KEY = "test_key_{}".format(randint(1, 2 ** 53 - 1)) - - -@pytest.fixture(autouse=True, scope="class") -def row_columns(): - global ROW_COLUMNS - ROW_COLUMNS = {"col1": "val1"} - - -class TestDatabases: - @pytest.fixture(scope="class") - def databases(self): - yield raw.get_databases() - - def test_create_databases(self): - response = raw.create_databases([DB_NAME]) - assert isinstance(response, RawResponse) - assert response.to_json()[0]["dbName"] == DB_NAME - - def test_databases_response_length(self, databases): - assert len(databases.to_json()) > 0 - - def test_get_databases_output_formats(self, databases): - assert isinstance(databases, RawResponse) - assert isinstance(databases.to_json(), list) - assert isinstance(databases.to_pandas(), pd.DataFrame) - - def test_delete_databases(self): - response = raw.delete_databases([DB_NAME], recursive=True) - assert response is None - with pytest.raises(APIError) as e: - raw.delete_databases([DB_NAME]) - - -class TestTables: - @pytest.fixture(autouse=True, scope="class") - def create_database(self): - raw.create_databases([DB_NAME]) - yield - raw.delete_databases([DB_NAME], recursive=True) - - @pytest.fixture(scope="class") - def tables(self): - yield raw.get_tables(DB_NAME) - - def test_create_tables(self): - response = raw.create_tables(DB_NAME, [TABLE_NAME]) - # assert isinstance(response, RawObject) - assert response.to_json()[0]["tableName"] == TABLE_NAME - - def test_tables_response_length(self, tables): - assert len(tables.to_json()) > 0 - - def test_tables_object_output_formats(self, tables): - assert isinstance(tables, RawResponse) - assert isinstance(tables.to_json(), list) - assert isinstance(tables.to_pandas(), pd.DataFrame) - - def test_delete_tables(self): - response = raw.delete_tables(database_name=DB_NAME, table_names=[TABLE_NAME]) - assert response is None - with pytest.raises(APIError) as e: - raw.delete_tables(DB_NAME, [TABLE_NAME]) - # assert re.match("{'code': 404, 'message': 'Did not find any dbs with the given names'}", str(e.value)) - # assert re.match("{'code': 404, 'message': 'No tables named test_table'}") - - -class TestRows: - @pytest.fixture(autouse=True, scope="class") - def create_database(self): - raw.create_databases([DB_NAME]) - raw.create_tables(DB_NAME, [TABLE_NAME]) - yield - raw.delete_databases([DB_NAME], recursive=True) - - def test_create_rows(self): - response = raw.create_rows(DB_NAME, TABLE_NAME, rows=[RawRow(key=ROW_KEY, columns=ROW_COLUMNS)]) - assert response is None - - def test_rows_response_length(self): - rows = raw.get_rows(database_name=DB_NAME, table_name=TABLE_NAME).to_json() - assert len(rows) == 1 - - def test_rows_object_output_formats(self): - row = raw.get_row(DB_NAME, TABLE_NAME, ROW_KEY) - assert isinstance(row, RawResponse) - assert isinstance(row.to_json(), list) - assert isinstance(row.to_pandas(), pd.DataFrame) - - def test_delete_rows(self): - response = raw.delete_rows(DB_NAME, TABLE_NAME, [RawRow(key=ROW_KEY, columns=ROW_COLUMNS)]) - assert response is None diff --git a/tests/test_stable/test_tag_matching.py b/tests/test_stable/test_tag_matching.py deleted file mode 100644 index 3b65ad8e04..0000000000 --- a/tests/test_stable/test_tag_matching.py +++ /dev/null @@ -1,55 +0,0 @@ -# Temporary mock setup while waiting for a better way to do integration tests -from unittest.mock import patch - -import pytest - -from cognite.client import CogniteClient -from cognite.client.stable.tagmatching import TagMatchingResponse -from tests.conftest import MockReturnValue - -tag_matching = CogniteClient().tag_matching.tag_matching - - -@pytest.fixture(scope="module") -@patch("requests.sessions.Session.post") -def tagmatching_result(mock_post): - response = { - "data": { - "items": [ - { - "matches": [ - {"platform": "a_platform", "score": 0, "tagId": "a_match"}, - {"platform": "a_platform", "score": 0, "tagId": "a_match1"}, - {"platform": "a_platform", "score": 0, "tagId": "a_match2"}, - ], - "tagId": "a_tag", - } - ] - } - } - mock_post.return_value = MockReturnValue(status=200, json_data=response) - return tag_matching(tag_ids=["a_tag"]) - - -def test_object(tagmatching_result): - assert isinstance(tagmatching_result, TagMatchingResponse) - - -def test_json(tagmatching_result): - assert isinstance(tagmatching_result.to_json(), list) - - -def test_pandas(tagmatching_result): - import pandas as pd - - assert isinstance(tagmatching_result.to_pandas(), pd.DataFrame) - - -def test_list_len_first_matches(tagmatching_result): - l = tagmatching_result.to_list(first_matches_only=False) - assert len(l) == 3 - - -def test_list_len_all_matches(tagmatching_result): - l = tagmatching_result.to_list() - assert len(l) == 1 diff --git a/tests/test_stable/test_timeseries.py b/tests/test_stable/test_timeseries.py deleted file mode 100644 index 7d24ff6016..0000000000 --- a/tests/test_stable/test_timeseries.py +++ /dev/null @@ -1,66 +0,0 @@ -from random import randint -from time import sleep - -import pandas as pd -import pytest - -from cognite.client import CogniteClient -from cognite.client.stable.time_series import TimeSeries, TimeSeriesListResponse, TimeSeriesResponse - -timeseries = CogniteClient().time_series - - -@pytest.fixture(autouse=True, scope="class") -def ts_name(): - global TS_NAME - TS_NAME = "test_ts_{}".format(randint(1, 2 ** 53 - 1)) - - -class TestTimeseries: - def test_post_timeseries(self): - tso = TimeSeries(TS_NAME) - res = timeseries.post_time_series([tso]) - assert res is None - - def test_update_timeseries(self): - tso = TimeSeries(TS_NAME, unit="celsius") - res = timeseries.update_time_series([tso]) - assert res is None - - @pytest.fixture(scope="class", params=[True, False]) - def get_timeseries_response_obj(self, request): - res = timeseries.get_time_series(prefix=TS_NAME, limit=1, include_metadata=request.param) - while len(res) == 0: - res = timeseries.get_time_series(prefix=TS_NAME, limit=1, include_metadata=request.param) - sleep(0.5) - yield res - - def test_timeseries_unit_correct(self, get_timeseries_response_obj): - assert get_timeseries_response_obj[0].unit == "celsius" - - def test_get_timeseries_output_format(self, get_timeseries_response_obj): - assert isinstance(get_timeseries_response_obj, TimeSeriesListResponse) - assert isinstance(get_timeseries_response_obj.to_pandas(), pd.DataFrame) - assert isinstance(get_timeseries_response_obj.to_json()[0], dict) - assert isinstance(get_timeseries_response_obj[0], TimeSeriesResponse) - assert isinstance(get_timeseries_response_obj[:1], TimeSeriesListResponse) - assert len(get_timeseries_response_obj[:1]) == 1 - - for ts in get_timeseries_response_obj: - assert isinstance(ts, TimeSeriesResponse) - assert isinstance(ts.to_pandas(), pd.DataFrame) - assert isinstance(ts.to_json(), dict) - for key, val in ts.__dict__.items(): - if key in ["metadata", "asset_id", "description"]: - assert val is None - else: - assert val is not None - - def test_get_timeseries_no_results(self): - result = timeseries.get_time_series(prefix="not_a_timeseries_prefix") - assert result.to_pandas().empty - assert not result.to_json() - - def test_delete_timeseries(self): - res = timeseries.delete_time_series(TS_NAME) - assert res is None diff --git a/tests/test_experimental/source_package_for_tests/my_model/__init__.py b/tests/tests_integration/__init__.py similarity index 100% rename from tests/test_experimental/source_package_for_tests/my_model/__init__.py rename to tests/tests_integration/__init__.py diff --git a/tests/tests_integration/test_api/test_assets.py b/tests/tests_integration/test_api/test_assets.py new file mode 100644 index 0000000000..8e91705ac3 --- /dev/null +++ b/tests/tests_integration/test_api/test_assets.py @@ -0,0 +1,99 @@ +import time + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes import Asset, AssetFilter, AssetUpdate +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils import _utils +from tests.utils import set_request_limit + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture +def new_asset(): + ts = COGNITE_CLIENT.assets.create(Asset(name="any")) + yield ts + COGNITE_CLIENT.assets.delete(id=ts.id) + assert COGNITE_CLIENT.assets.retrieve(ts.id) is None + + +def generate_asset_tree(root_external_id: str, depth: int, children_per_node: int, current_depth=1): + assert 1 <= children_per_node <= 10, "children_per_node must be between 1 and 10" + assets = [] + if current_depth == 1: + assets = [Asset(external_id=root_external_id, name=root_external_id)] + if depth > current_depth: + for i in range(children_per_node): + external_id = "{}{}".format(root_external_id, i) + asset = Asset(parent_external_id=root_external_id, external_id=external_id, name=external_id) + assets.append(asset) + if depth > current_depth + 1: + assets.extend( + generate_asset_tree(root_external_id + str(i), depth, children_per_node, current_depth + 1) + ) + return assets + + +@pytest.fixture +def new_asset_hierarchy(mocker): + random_prefix = "test_{}_".format(_utils.random_string(10)) + assets = generate_asset_tree(random_prefix + "0", depth=5, children_per_node=5) + mocker.spy(COGNITE_CLIENT.assets, "_post") + + with set_request_limit(COGNITE_CLIENT.assets, 50): + COGNITE_CLIENT.assets.create(assets) + + assert 20 < COGNITE_CLIENT.assets._post.call_count < 30 + + ext_ids = [a.external_id for a in assets] + yield random_prefix, ext_ids + + COGNITE_CLIENT.assets.delete(external_id=random_prefix + "0") + + +@pytest.fixture +def root_test_asset(): + for asset in COGNITE_CLIENT.assets(root=True): + if asset.name.startswith("test__"): + return asset + + +class TestAssetsAPI: + def test_get(self): + res = COGNITE_CLIENT.assets.list(limit=1) + assert res[0] == COGNITE_CLIENT.assets.retrieve(res[0].id) + + def test_list(self, mocker): + mocker.spy(COGNITE_CLIENT.assets, "_post") + + with set_request_limit(COGNITE_CLIENT.assets, 10): + res = COGNITE_CLIENT.assets.list(limit=20) + + assert 20 == len(res) + assert 2 == COGNITE_CLIENT.assets._post.call_count + + def test_search(self): + res = COGNITE_CLIENT.assets.search(name="test__asset_0", filter=AssetFilter(name="test__asset_0")) + assert len(res) > 0 + + def test_update(self, new_asset): + update_asset = AssetUpdate(new_asset.id).name.set("newname") + res = COGNITE_CLIENT.assets.update(update_asset) + assert "newname" == res.name + + def test_post_asset_hierarchy(self, new_asset_hierarchy): + prefix, ext_ids = new_asset_hierarchy + posted_assets = COGNITE_CLIENT.assets.retrieve_multiple(external_ids=ext_ids) + external_id_to_id = {a.external_id: a.id for a in posted_assets} + + for asset in posted_assets: + if asset.external_id == prefix + "0": + assert asset.parent_id is None + else: + assert asset.parent_id == external_id_to_id[asset.external_id[:-1]] + + def test_get_subtree(self, root_test_asset): + assert 781 == len(COGNITE_CLIENT.assets.retrieve_subtree(root_test_asset.id)) + assert 6 == len(COGNITE_CLIENT.assets.retrieve_subtree(root_test_asset.id, depth=1)) diff --git a/tests/tests_integration/test_api/test_datapoints.py b/tests/tests_integration/test_api/test_datapoints.py new file mode 100644 index 0000000000..5d123924ca --- /dev/null +++ b/tests/tests_integration/test_api/test_datapoints.py @@ -0,0 +1,131 @@ +import re +from datetime import datetime, timedelta + +import numpy +import pandas +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes import Datapoint, DatapointsQuery, TimeSeries +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils import _utils +from tests.utils import set_request_limit + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture(scope="session") +def test_time_series(): + time_series = {} + for ts in COGNITE_CLIENT.time_series.list(limit=150): + if ts.name in ["test__constant_{}_with_noise".format(i) for i in range(0, 10)]: + value = int(re.match("test__constant_(\d+)_with_noise", ts.name).group(1)) + time_series[value] = ts + yield time_series + + +@pytest.fixture(scope="session") +def new_ts(): + ts = COGNITE_CLIENT.time_series.create(TimeSeries()) + yield ts + COGNITE_CLIENT.time_series.delete(id=ts.id) + assert COGNITE_CLIENT.time_series.retrieve(ts.id) is None + + +def has_duplicates(df: pandas.DataFrame): + return df.duplicated().any() + + +def has_correct_timestamp_spacing(df: pandas.DataFrame, granularity: str): + timestamps = df.index.values.astype("datetime64[ms]").astype("int64") + deltas = numpy.diff(timestamps, 1) + granularity_ms = _utils.granularity_to_ms(granularity) + return (deltas != 0).all() and (deltas % granularity_ms == 0).all() + + +class TestDatapointsAPI: + def test_retrieve(self, test_time_series): + ts = test_time_series[0] + dps = COGNITE_CLIENT.datapoints.retrieve(id=ts.id, start="1d-ago", end="now") + assert len(dps) > 0 + + def test_retrieve_multiple(self, test_time_series): + ids = [test_time_series[0].id, test_time_series[1].id, {"id": test_time_series[2].id, "aggregates": ["max"]}] + + dps = COGNITE_CLIENT.datapoints.retrieve( + id=ids, start="6h-ago", end="now", aggregates=["min"], granularity="1s" + ) + df = dps.to_pandas(column_names="id") + assert "{}|min".format(test_time_series[0].id) in df.columns + assert "{}|min".format(test_time_series[1].id) in df.columns + assert "{}|max".format(test_time_series[2].id) in df.columns + assert 0 < df.shape[0] + assert 3 == df.shape[1] + assert has_correct_timestamp_spacing(df, "1s") + + def test_retrieve_include_outside_points(self, test_time_series): + ts = test_time_series[0] + start = _utils.timestamp_to_ms("6h-ago") + end = _utils.timestamp_to_ms("1h-ago") + dps_wo_outside = COGNITE_CLIENT.datapoints.retrieve( + id=ts.id, start=start, end=end, include_outside_points=False + ) + dps_w_outside = COGNITE_CLIENT.datapoints.retrieve(id=ts.id, start=start, end=end, include_outside_points=True) + assert not has_duplicates(dps_w_outside.to_pandas()) + assert len(dps_wo_outside) + 1 <= len(dps_w_outside) <= len(dps_wo_outside) + 2 + + def test_retrieve_dataframe(self, test_time_series): + ts = test_time_series[0] + df = COGNITE_CLIENT.datapoints.retrieve_dataframe( + id=ts.id, start="6h-ago", end="now", aggregates=["average"], granularity="1s" + ) + assert df.shape[0] > 0 + assert df.shape[1] == 1 + assert has_correct_timestamp_spacing(df, "1s") + + def test_query(self, test_time_series): + dps_query1 = DatapointsQuery(id=test_time_series[0].id, start="6h-ago", end="now") + dps_query2 = DatapointsQuery(id=test_time_series[1].id, start="3h-ago", end="now") + dps_query3 = DatapointsQuery( + id=test_time_series[2].id, start="1d-ago", end="now", aggregates=["average"], granularity="1h" + ) + + res = COGNITE_CLIENT.datapoints.query([dps_query1, dps_query2, dps_query3]) + assert len(res) == 3 + assert ( + len(res.get(test_time_series[2].id)) + < len(res.get(test_time_series[1].id)) + < len(res.get(test_time_series[0].id)) + ) + + def test_retrieve_latest(self, test_time_series): + ids = [test_time_series[0].id, test_time_series[1].id] + res = COGNITE_CLIENT.datapoints.retrieve_latest(id=ids) + for dps in res: + assert 1 == len(dps) + + def test_retrieve_latest_before(self, test_time_series): + ts = test_time_series[0] + res = COGNITE_CLIENT.datapoints.retrieve_latest(id=ts.id, before="1h-ago") + assert 1 == len(res) + assert res[0].timestamp < _utils.timestamp_to_ms("1h-ago") + + def test_insert(self, new_ts, mocker): + datapoints = [(datetime(year=2018, month=1, day=1, hour=1, minute=i), i) for i in range(60)] + mocker.spy(COGNITE_CLIENT.datapoints, "_post") + with set_request_limit(COGNITE_CLIENT.datapoints, 30): + COGNITE_CLIENT.datapoints.insert(datapoints, id=new_ts.id) + assert 2 == COGNITE_CLIENT.datapoints._post.call_count + + def test_insert_pandas_dataframe(self, new_ts, mocker): + start = datetime(2018, 1, 1) + x = pandas.DatetimeIndex([start + timedelta(days=d) for d in range(100)]) + y = numpy.random.normal(0, 1, 100) + df = pandas.DataFrame({new_ts.id: y}, index=x) + mocker.spy(COGNITE_CLIENT.datapoints, "_post") + with set_request_limit(COGNITE_CLIENT.datapoints, 50): + COGNITE_CLIENT.datapoints.insert_dataframe(df) + assert 2 == COGNITE_CLIENT.datapoints._post.call_count + + def test_delete_range(self, new_ts): + COGNITE_CLIENT.datapoints.delete_range(start="2d-ago", end="now", id=new_ts.id) diff --git a/tests/tests_integration/test_api/test_events.py b/tests/tests_integration/test_api/test_events.py new file mode 100644 index 0000000000..54cb325b53 --- /dev/null +++ b/tests/tests_integration/test_api/test_events.py @@ -0,0 +1,44 @@ +import pytest + +from cognite.client import CogniteClient, utils +from cognite.client.data_classes import Event, EventFilter, EventUpdate +from cognite.client.exceptions import CogniteAPIError +from tests.utils import set_request_limit + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture +def new_event(): + event = COGNITE_CLIENT.events.create(Event()) + yield event + COGNITE_CLIENT.events.delete(id=event.id) + assert COGNITE_CLIENT.events.retrieve(event.id) is None + + +class TestEventsAPI: + def test_retrieve(self): + res = COGNITE_CLIENT.events.list(limit=1) + assert res[0] == COGNITE_CLIENT.events.retrieve(res[0].id) + + def test_retrieve_multiple(self): + res = COGNITE_CLIENT.events.list(limit=2) + assert res == COGNITE_CLIENT.events.retrieve_multiple([e.id for e in res]) + + def test_list(self, mocker): + mocker.spy(COGNITE_CLIENT.events, "_post") + + with set_request_limit(COGNITE_CLIENT.events, 10): + res = COGNITE_CLIENT.events.list(limit=20) + + assert 20 == len(res) + assert 2 == COGNITE_CLIENT.events._post.call_count + + def test_search(self): + res = COGNITE_CLIENT.events.search(filter=EventFilter(start_time={"min": utils.timestamp_to_ms("2d-ago")})) + assert len(res) > 0 + + def test_update(self, new_event): + update_asset = EventUpdate(new_event.id).metadata.set({"bla": "bla"}) + res = COGNITE_CLIENT.events.update(update_asset) + assert {"bla": "bla"} == res.metadata diff --git a/tests/tests_integration/test_api/test_files.py b/tests/tests_integration/test_api/test_files.py new file mode 100644 index 0000000000..c7719cc8c9 --- /dev/null +++ b/tests/tests_integration/test_api/test_files.py @@ -0,0 +1,52 @@ +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes import FileMetadata, FileMetadataFilter, FileMetadataUpdate +from cognite.client.exceptions import CogniteAPIError + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture(scope="class") +def new_file(): + res = COGNITE_CLIENT.files.upload_bytes(content="blabla", name="myspecialfile") + yield res + COGNITE_CLIENT.files.delete(id=res.id) + assert COGNITE_CLIENT.files.retrieve(id=res.id) is None + + +@pytest.fixture(scope="class") +def test_files(): + files = {} + for file in COGNITE_CLIENT.files: + if file.name in ["a.txt", "b.txt", "c.txt", "big.txt"]: + files[file.name] = file + return files + + +class TestFilesAPI: + def test_retrieve(self): + res = COGNITE_CLIENT.files.list(limit=1) + assert res[0] == COGNITE_CLIENT.files.retrieve(res[0].id) + + def test_retrieve_multiple(self): + res = COGNITE_CLIENT.files.list(limit=2) + assert res == COGNITE_CLIENT.files.retrieve_multiple([f.id for f in res]) + + def test_list(self): + res = COGNITE_CLIENT.files.list(limit=4) + assert 4 == len(res) + + def test_search(self): + res = COGNITE_CLIENT.files.search(name="big.txt", filter=FileMetadataFilter(created_time={"min": 0})) + assert len(res) > 0 + + def test_update(self, new_file): + update_file = FileMetadataUpdate(new_file.id).metadata.set({"bla": "bla"}) + res = COGNITE_CLIENT.files.update(update_file) + assert {"bla": "bla"} == res.metadata + + def test_download(self, test_files): + test_file = test_files["a.txt"] + res = COGNITE_CLIENT.files.download_bytes(id=test_file.id) + assert b"a" == res diff --git a/tests/tests_integration/test_api/test_iam.py b/tests/tests_integration/test_api/test_iam.py new file mode 100644 index 0000000000..56ca827787 --- /dev/null +++ b/tests/tests_integration/test_api/test_iam.py @@ -0,0 +1,83 @@ +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes import APIKey, Group, SecurityCategory, ServiceAccount, ServiceAccountList +from cognite.client.utils._utils import random_string + +COGNITE_CLIENT = CogniteClient() + + +class TestServiceAccountAPI: + def test_list(self): + res = COGNITE_CLIENT.iam.service_accounts.list() + assert isinstance(res, ServiceAccountList) + assert len(res) > 0 + + def test_create_and_delete(self): + name = "test_sa_" + random_string(10) + sa = COGNITE_CLIENT.iam.service_accounts.create(ServiceAccount(name=name)) + assert isinstance(sa, ServiceAccount) + assert sa.id in {s.id for s in COGNITE_CLIENT.iam.service_accounts.list()} + COGNITE_CLIENT.iam.service_accounts.delete(sa.id) + assert sa.id not in {s.id for s in COGNITE_CLIENT.iam.service_accounts.list()} + + +@pytest.fixture(scope="module") +def service_account_id(): + return COGNITE_CLIENT.iam.service_accounts.list()[0].id + + +class TestAPIKeysAPI: + def test_list_with_sa_id(self, service_account_id): + res = COGNITE_CLIENT.iam.api_keys.list(service_account_id=service_account_id) + assert len(res) > 0 + + def test_list_all(self): + res = COGNITE_CLIENT.iam.api_keys.list(all=True) + assert len(res) > 0 + + def test_list_deleted(self): + res = COGNITE_CLIENT.iam.api_keys.list(include_deleted=True) + assert len(res) > 0 + + def test_create_and_delete(self, service_account_id): + res = COGNITE_CLIENT.iam.api_keys.create(service_account_id) + assert isinstance(res, APIKey) + assert res.id in {k.id for k in COGNITE_CLIENT.iam.api_keys.list(all=True)} + COGNITE_CLIENT.iam.api_keys.delete(res.id) + assert res.id not in {k.id for k in COGNITE_CLIENT.iam.api_keys.list(all=True)} + + +@pytest.fixture(scope="module") +def group_id(): + return COGNITE_CLIENT.iam.groups.list()[0].id + + +class TestGroupsAPI: + @pytest.mark.xfail(strict=True) + def test_list(self): + res = COGNITE_CLIENT.iam.groups.list(all=True) + assert len(res) > 0 + + @pytest.mark.xfail(strict=True) + def test_create_and_delete(self): + group = COGNITE_CLIENT.iam.groups.create(Group(name="bla")) + COGNITE_CLIENT.iam.groups.delete(group.id) + assert group.id not in {g.id for g in COGNITE_CLIENT.iam.groups.list(all=True)} + + def test_list_service_accounts_in_group(self, group_id): + service_accounts = COGNITE_CLIENT.iam.groups.list_service_accounts(group_id) + assert len(service_accounts) > 0 + + +class TestSecurityCategoriesAPI: + def test_list(self): + res = COGNITE_CLIENT.iam.security_categories.list() + assert len(res) > 0 + + def test_create_and_delete(self): + random_name = "test_" + random_string(10) + res = COGNITE_CLIENT.iam.security_categories.create(SecurityCategory(name=random_name)) + assert res.id in {s.id for s in COGNITE_CLIENT.iam.security_categories.list()} + COGNITE_CLIENT.iam.security_categories.delete(res.id) + assert res.id not in {s.id for s in COGNITE_CLIENT.iam.security_categories.list()} diff --git a/tests/tests_integration/test_api/test_login.py b/tests/tests_integration/test_api/test_login.py new file mode 100644 index 0000000000..f5e4a54197 --- /dev/null +++ b/tests/tests_integration/test_api/test_login.py @@ -0,0 +1,14 @@ +from cognite.client import CogniteClient + +c = CogniteClient() + + +class TestLoginAPI: + def test_login_status(self): + assert { + "user": "python-sdk-integration-tester", + "project": "python-sdk-test", + "project_id": 2561337318642649, + "logged_in": True, + "api_key_id": 4131947729676274, + } == c.login.status().dump() diff --git a/tests/tests_integration/test_api/test_raw.py b/tests/tests_integration/test_api/test_raw.py new file mode 100644 index 0000000000..2aa282c01f --- /dev/null +++ b/tests/tests_integration/test_api/test_raw.py @@ -0,0 +1,60 @@ +import time + +import pytest + +from cognite.client import CogniteClient +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils import _utils + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture(scope="session") +def new_database_with_table(): + db_name = "db_" + _utils.random_string(10) + table_name = "table_" + _utils.random_string(10) + db = COGNITE_CLIENT.raw.databases.create(db_name) + table = COGNITE_CLIENT.raw.tables.create(db_name, table_name) + yield db, table + COGNITE_CLIENT.raw.databases.delete(db.name) + + +class TestRawDatabasesAPI: + def test_list_databases(self): + dbs = COGNITE_CLIENT.raw.databases.list() + assert len(dbs) > 0 + + def test_create_and_delete_database(self, new_database_with_table): + pass + + +class TestRawTablesAPI: + def test_list_tables(self): + tables = COGNITE_CLIENT.raw.tables.list(db_name="test__database1") + assert len(tables) == 3 + + def test_create_and_delete_table(self, new_database_with_table): + db, _ = new_database_with_table + table_name = "table_" + _utils.random_string(10) + table = COGNITE_CLIENT.raw.tables.create(db.name, table_name) + assert table in COGNITE_CLIENT.raw.tables.list(db.name) + COGNITE_CLIENT.raw.tables.delete(db.name, table.name) + assert not table in COGNITE_CLIENT.raw.tables.list(db.name) + + +class TestRawRowsAPI: + def test_list_rows(self): + rows = COGNITE_CLIENT.raw.rows.list(db_name="test__database1", table_name="test__table_1", limit=-1) + assert 2000 == len(rows) + + def test_retrieve_row(self): + row = COGNITE_CLIENT.raw.rows.retrieve(db_name="test__database1", table_name="test__table_1", key="1") + assert {"c{}".format(i): "1_{}".format(i) for i in range(10)} == row.columns + + def test_insert_and_delete_rows(self, new_database_with_table): + db, table = new_database_with_table + rows = {"r1": {"c1": "v1", "c2": "v1"}, "r2": {"c1": "v2", "c2": "v2"}} + COGNITE_CLIENT.raw.rows.insert(db.name, table.name, rows) + assert 2 == len(table.rows()) + COGNITE_CLIENT.raw.rows.delete(db.name, table.name, ["r1", "r2"]) + assert 0 == len(table.rows()) diff --git a/tests/tests_integration/test_api/test_time_series.py b/tests/tests_integration/test_api/test_time_series.py new file mode 100644 index 0000000000..1c1a3e6b24 --- /dev/null +++ b/tests/tests_integration/test_api/test_time_series.py @@ -0,0 +1,68 @@ +import time + +import pytest + +from cognite.client import CogniteClient, utils +from cognite.client.data_classes import TimeSeries, TimeSeriesFilter, TimeSeriesUpdate +from cognite.client.exceptions import CogniteAPIError +from tests.utils import set_request_limit + +COGNITE_CLIENT = CogniteClient() + + +@pytest.fixture(scope="class") +def new_ts(): + ts = COGNITE_CLIENT.time_series.create(TimeSeries(name="any")) + yield ts + COGNITE_CLIENT.time_series.delete(id=ts.id) + assert COGNITE_CLIENT.time_series.retrieve(ts.id) is None + + +class TestTimeSeriesAPI: + def test_retrieve(self): + listed_asset = COGNITE_CLIENT.time_series.list(limit=1)[0] + retrieved_asset = COGNITE_CLIENT.time_series.retrieve(listed_asset.id) + retrieved_asset.external_id = listed_asset.external_id + assert retrieved_asset == listed_asset + + def test_retrieve_multiple(self): + res = COGNITE_CLIENT.time_series.list(limit=2) + retrieved_assets = COGNITE_CLIENT.time_series.retrieve_multiple([t.id for t in res]) + for listed_asset, retrieved_asset in zip(res, retrieved_assets): + retrieved_asset.external_id = listed_asset.external_id + assert res == retrieved_assets + + def test_list(self, mocker): + mocker.spy(COGNITE_CLIENT.time_series, "_get") + + with set_request_limit(COGNITE_CLIENT.time_series, 10): + res = COGNITE_CLIENT.time_series.list(limit=20) + + assert 20 == len(res) + assert 2 == COGNITE_CLIENT.time_series._get.call_count + + def test_search(self): + res = COGNITE_CLIENT.time_series.search( + name="test__timestamp_multiplied", filter=TimeSeriesFilter(created_time={"min": 0}) + ) + assert len(res) > 0 + + def test_update(self, new_ts): + update_ts = TimeSeriesUpdate(new_ts.id).name.set("newname") + res = COGNITE_CLIENT.time_series.update(update_ts) + assert "newname" == res.name + + def test_list_created_ts(self, new_ts): + tries = 8 + sleep_between_tries = 3 + found = False + for i in range(tries): + for ts in COGNITE_CLIENT.time_series: + if ts.id == new_ts.id: + found = True + break + if found: + break + elif i < tries - 1: + time.sleep(sleep_between_tries) + assert found is True diff --git a/tests/tests_integration/test_cognite_client.py b/tests/tests_integration/test_cognite_client.py new file mode 100644 index 0000000000..73871521c3 --- /dev/null +++ b/tests/tests_integration/test_cognite_client.py @@ -0,0 +1,27 @@ +import pytest + +from cognite.client import CogniteClient +from cognite.client.exceptions import CogniteAPIError + +c = CogniteClient() + + +class TestCogniteClient: + def test_get(self): + res = c.get("/login/status") + assert res.status_code == 200 + + def test_post(self): + with pytest.raises(CogniteAPIError) as e: + c.post("/login", json={}) + assert e.value.code == 404 + + def test_put(self): + with pytest.raises(CogniteAPIError) as e: + c.put("/login") + assert e.value.code == 404 + + def test_delete(self): + with pytest.raises(CogniteAPIError) as e: + c.delete("/login") + assert e.value.code == 404 diff --git a/tests/tests_unit/__init__.py b/tests/tests_unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_api/__init__.py b/tests/tests_unit/test_api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_1.txt b/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_1.txt new file mode 100644 index 0000000000..ac3e272b72 --- /dev/null +++ b/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_1.txt @@ -0,0 +1 @@ +content1 diff --git a/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_2.txt b/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_2.txt new file mode 100644 index 0000000000..637f0347d3 --- /dev/null +++ b/tests/tests_unit/test_api/files_for_test_upload/file_for_test_upload_2.txt @@ -0,0 +1 @@ +content2 diff --git a/tests/tests_unit/test_api/files_for_test_upload/files_for_test_recursive_upload/file_for_test_upload_3.txt b/tests/tests_unit/test_api/files_for_test_upload/files_for_test_recursive_upload/file_for_test_upload_3.txt new file mode 100644 index 0000000000..27d10cc8d0 --- /dev/null +++ b/tests/tests_unit/test_api/files_for_test_upload/files_for_test_recursive_upload/file_for_test_upload_3.txt @@ -0,0 +1 @@ +content3 diff --git a/tests/tests_unit/test_api/test_3d.py b/tests/tests_unit/test_api/test_3d.py new file mode 100644 index 0000000000..9612e74ff6 --- /dev/null +++ b/tests/tests_unit/test_api/test_3d.py @@ -0,0 +1,376 @@ +import re + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.three_d import ( + ThreeDAssetMapping, + ThreeDAssetMappingList, + ThreeDModel, + ThreeDModelList, + ThreeDModelRevision, + ThreeDModelRevisionList, + ThreeDModelRevisionUpdate, + ThreeDModelUpdate, + ThreeDNodeList, + ThreeDRevealNodeList, + ThreeDRevealRevision, + ThreeDRevealSectorList, +) +from cognite.client.exceptions import CogniteAPIError +from tests.utils import jsgz_load + +THREE_D_API = CogniteClient().three_d + + +@pytest.fixture +def mock_3d_model_response(rsps): + response_body = {"items": [{"name": "My Model", "id": 1000, "createdTime": 0}]} + url_pattern = re.compile(re.escape(THREE_D_API._base_url) + "/3d/models.*") + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + rsps.assert_all_requests_are_fired = False + yield rsps + + +@pytest.fixture +def mock_retrieve_3d_model_response(rsps): + response_body = {"name": "My Model", "id": 1000, "createdTime": 0} + rsps.add(rsps.GET, THREE_D_API._base_url + "/3d/models/1", status=200, json=response_body) + yield rsps + + +class Test3DModels: + def test_list(self, mock_3d_model_response): + res = THREE_D_API.models.list(published=True, limit=100) + assert isinstance(res, ThreeDModelList) + assert mock_3d_model_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_update_with_update_object(self, mock_3d_model_response): + update = ThreeDModelUpdate(id=1).name.set("bla") + res = THREE_D_API.models.update(update) + assert {"id": 1, "update": {"name": {"set": "bla"}}} == jsgz_load(mock_3d_model_response.calls[0].request.body)[ + "items" + ][0] + assert mock_3d_model_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_with_resource_object(self, mock_3d_model_response): + res = THREE_D_API.models.update(ThreeDModel(id=1, name="bla", created_time=123)) + assert {"id": 1, "update": {"name": {"set": "bla"}}} == jsgz_load(mock_3d_model_response.calls[0].request.body)[ + "items" + ][0] + assert mock_3d_model_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_delete(self, mock_3d_model_response): + res = THREE_D_API.models.delete(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_3d_model_response.calls[0].request.body) + assert res is None + res = THREE_D_API.models.delete(id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_3d_model_response.calls[1].request.body) + assert res is None + + def test_retrieve(self, mock_retrieve_3d_model_response): + res = THREE_D_API.models.retrieve(id=1) + assert isinstance(res, ThreeDModel) + assert mock_retrieve_3d_model_response.calls[0].response.json() == res.dump(camel_case=True) + + def test_create(self, mock_3d_model_response): + res = THREE_D_API.models.create(name="My Model") + assert isinstance(res, ThreeDModel) + assert jsgz_load(mock_3d_model_response.calls[0].request.body) == {"items": [{"name": "My Model"}]} + assert mock_3d_model_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_3d_model_response): + res = THREE_D_API.models.create(name=["My Model"]) + assert isinstance(res, ThreeDModelList) + assert jsgz_load(mock_3d_model_response.calls[0].request.body) == {"items": [{"name": "My Model"}]} + assert mock_3d_model_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + +@pytest.fixture +def mock_3d_model_revision_response(rsps): + response_body = { + "items": [ + { + "id": 1, + "fileId": 1000, + "published": False, + "rotation": [0, 0, 0], + "camera": {"target": [0, 0, 0], "position": [0, 0, 0]}, + "status": "Done", + "thumbnailThreedFileId": 1000, + "thumbnailUrl": "https://api.cognitedata.com/api/v1/project/myproject/3d/files/1000", + "assetMappingCount": 0, + "createdTime": 0, + } + ] + } + url_pattern = re.compile(re.escape(THREE_D_API._base_url) + "/3d/models/1/revisions.*") + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + rsps.assert_all_requests_are_fired = False + yield rsps + + +@pytest.fixture +def mock_retrieve_3d_model_revision_response(rsps): + res = { + "id": 1000, + "fileId": 1000, + "published": False, + "rotation": [0, 0, 0], + "camera": {"target": [0, 0, 0], "position": [0, 0, 0]}, + "status": "Done", + "thumbnailThreedFileId": 1000, + "thumbnailUrl": "https://api.cognitedata.com/api/v1/project/myproject/3d/files/1000", + "assetMappingCount": 0, + "createdTime": 0, + } + rsps.add(rsps.GET, THREE_D_API._base_url + "/3d/models/1/revisions/1", status=200, json=res) + yield rsps + + +@pytest.fixture +def mock_3d_model_revision_thumbnail_response(rsps): + rsps.add(rsps.POST, THREE_D_API._base_url + "/3d/models/1/revisions/1/thumbnail", status=200, json={}) + yield rsps + + +@pytest.fixture +def mock_3d_model_revision_node_response(rsps): + response_body = { + "items": [ + { + "id": 1, + "treeIndex": 3, + "parentId": 2, + "depth": 2, + "name": "Node name", + "subtreeSize": 4, + "boundingBox": {"max": [0, 0, 0], "min": [0, 0, 0]}, + } + ] + } + rsps.add(rsps.GET, THREE_D_API._base_url + "/3d/models/1/revisions/1/nodes", status=200, json=response_body) + rsps.add( + rsps.GET, THREE_D_API._base_url + "/3d/models/1/revisions/1/nodes/ancestors", status=200, json=response_body + ) + rsps.assert_all_requests_are_fired = False + yield rsps + + +class Test3DModelRevisions: + def test_list(self, mock_3d_model_revision_response): + res = THREE_D_API.revisions.list(model_id=1, published=True, limit=100) + assert isinstance(res, ThreeDModelRevisionList) + assert mock_3d_model_revision_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_update_with_update_object(self, mock_3d_model_revision_response): + update = ThreeDModelRevisionUpdate(id=1).published.set(False) + THREE_D_API.revisions.update(1, update) + assert {"id": 1, "update": {"published": {"set": False}}} == jsgz_load( + mock_3d_model_revision_response.calls[0].request.body + )["items"][0] + + def test_update_with_resource_object(self, mock_3d_model_revision_response): + THREE_D_API.revisions.update(1, ThreeDModelRevision(id=1, published=False, created_time=123)) + assert {"id": 1, "update": {"published": {"set": False}}} == jsgz_load( + mock_3d_model_revision_response.calls[0].request.body + )["items"][0] + + def test_delete(self, mock_3d_model_revision_response): + res = THREE_D_API.revisions.delete(1, id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_3d_model_revision_response.calls[0].request.body) + assert res is None + res = THREE_D_API.revisions.delete(1, id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_3d_model_revision_response.calls[1].request.body) + assert res is None + + def test_retrieve(self, mock_retrieve_3d_model_revision_response): + res = THREE_D_API.revisions.retrieve(model_id=1, id=1) + assert isinstance(res, ThreeDModelRevision) + assert mock_retrieve_3d_model_revision_response.calls[0].response.json() == res.dump(camel_case=True) + + def test_create(self, mock_3d_model_revision_response): + res = THREE_D_API.revisions.create(model_id=1, revision=ThreeDModelRevision(file_id=123)) + assert isinstance(res, ThreeDModelRevision) + assert {"items": [{"fileId": 123}]} == jsgz_load(mock_3d_model_revision_response.calls[0].request.body) + assert mock_3d_model_revision_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_3d_model_revision_response): + res = THREE_D_API.revisions.create(model_id=1, revision=[ThreeDModelRevision(file_id=123)]) + assert isinstance(res, ThreeDModelRevisionList) + assert {"items": [{"fileId": 123}]} == jsgz_load(mock_3d_model_revision_response.calls[0].request.body) + assert mock_3d_model_revision_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_update_thumbnail(self, mock_3d_model_revision_thumbnail_response): + res = THREE_D_API.revisions.update_thumbnail(model_id=1, revision_id=1, file_id=1) + assert {"fileId": 1} == jsgz_load(mock_3d_model_revision_thumbnail_response.calls[0].request.body) + assert res is None + + def test_list_3d_nodes(self, mock_3d_model_revision_node_response): + res = THREE_D_API.revisions.list_nodes(model_id=1, revision_id=1, node_id=None, depth=None, limit=10) + assert isinstance(res, ThreeDNodeList) + assert mock_3d_model_revision_node_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list_3d_ancestor_nodes(self, mock_3d_model_revision_node_response): + res = THREE_D_API.revisions.list_ancestor_nodes(model_id=1, revision_id=1, node_id=None, limit=10) + assert isinstance(res, ThreeDNodeList) + assert mock_3d_model_revision_node_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + +class Test3DFiles: + @pytest.fixture + def mock_3d_files_response(self, rsps): + rsps.add(rsps.GET, THREE_D_API._base_url + "/3d/files/1", body="bla") + + def test_retrieve(self, mock_3d_files_response): + assert b"bla" == THREE_D_API.files.retrieve(1) + + +class Test3DAssetMappings: + @pytest.fixture + def mock_3d_asset_mappings_response(self, rsps): + response_body = {"items": [{"nodeId": 1003, "assetId": 3001, "treeIndex": 5, "subtreeSize": 7}]} + url_pattern = re.compile(re.escape(THREE_D_API._base_url) + "/3d/models/1/revisions/1/mappings.*") + + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.assert_all_requests_are_fired = False + yield rsps + + def test_list(self, mock_3d_asset_mappings_response): + res = THREE_D_API.asset_mappings.list(model_id=1, revision_id=1, node_id=None, asset_id=None, limit=None) + assert isinstance(res, ThreeDAssetMappingList) + assert mock_3d_asset_mappings_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create(self, mock_3d_asset_mappings_response): + res = THREE_D_API.asset_mappings.create( + model_id=1, revision_id=1, asset_mapping=ThreeDAssetMapping(node_id=1, asset_id=1) + ) + assert isinstance(res, ThreeDAssetMapping) + assert mock_3d_asset_mappings_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_3d_asset_mappings_response): + res = THREE_D_API.asset_mappings.create( + model_id=1, revision_id=1, asset_mapping=[ThreeDAssetMapping(node_id=1, asset_id=1)] + ) + assert isinstance(res, ThreeDAssetMappingList) + assert mock_3d_asset_mappings_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_delete(self, mock_3d_asset_mappings_response): + res = THREE_D_API.asset_mappings.delete(model_id=1, revision_id=1, asset_mapping=ThreeDAssetMapping(1, 1)) + assert res is None + assert [{"nodeId": 1, "assetId": 1}] == jsgz_load(mock_3d_asset_mappings_response.calls[0].request.body)[ + "items" + ] + + def test_delete_multiple(self, mock_3d_asset_mappings_response): + res = THREE_D_API.asset_mappings.delete(model_id=1, revision_id=1, asset_mapping=[ThreeDAssetMapping(1, 1)]) + assert res is None + assert [{"nodeId": 1, "assetId": 1}] == jsgz_load(mock_3d_asset_mappings_response.calls[0].request.body)[ + "items" + ] + + def test_delete_fails(self, rsps): + rsps.add( + rsps.POST, + THREE_D_API._base_url + "/3d/models/1/revisions/1/mappings/delete", + status=500, + json={"error": {"message": "Server Error", "code": 500}}, + ) + with pytest.raises(CogniteAPIError) as e: + THREE_D_API.asset_mappings.delete(model_id=1, revision_id=1, asset_mapping=[ThreeDAssetMapping(1, 1)]) + assert e.value.unknown == [ThreeDAssetMapping._load({"assetId": 1, "nodeId": 1})] + + +@pytest.mark.skip +class Test3DReveal: + @pytest.fixture + def mock_get_reveal_revision_response(self, rsps): + res = { + "id": 1000, + "fileId": 1000, + "published": False, + "rotation": [0, 0, 0], + "camera": {"target": [0, 0, 0], "position": [0, 0, 0]}, + "status": "Done", + "thumbnailThreedFileId": 1000, + "thumbnailUrl": "https://api.cognitedata.com/api/v1/project/myproject/3d/files/1000", + "assetMappingCount": 0, + "createdTime": 0, + "sceneThreedFiles": [{"version": 1, "fileId": 1000}], + } + rsps.add(rsps.GET, THREE_D_API._base_url + "/3d/reveal/models/1/revisions/1", status=200, json=res) + yield rsps + + def test_retrieve_revision(self, mock_get_reveal_revision_response): + res = THREE_D_API.reveal.retrieve_revision(model_id=1, revision_id=1) + assert isinstance(res, ThreeDRevealRevision) + assert mock_get_reveal_revision_response.calls[0].response.json() == res.dump(camel_case=True) + + @pytest.fixture + def mock_list_reveal_nodes_response(self, rsps): + res = { + "items": [ + { + "id": 1000, + "treeIndex": 3, + "parentId": 2, + "depth": 2, + "name": "Node name", + "subtreeSize": 4, + "boundingBox": {"max": [0, 0, 0], "min": [0, 0, 0]}, + "sectorId": 1000, + } + ] + } + rsps.add( + rsps.GET, + re.compile(re.escape(THREE_D_API._base_url) + "/3d/reveal/models/1/revisions/1/nodes.*"), + status=200, + json=res, + ) + yield rsps + + def test_list_nodes(self, mock_list_reveal_nodes_response): + res = THREE_D_API.reveal.list_nodes(model_id=1, revision_id=1, node_id=None, depth=None, limit=None) + assert isinstance(res, ThreeDRevealNodeList) + assert mock_list_reveal_nodes_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list_ancestor_nodes(self, mock_list_reveal_nodes_response): + res = THREE_D_API.reveal.list_ancestor_nodes(model_id=1, revision_id=1, node_id=None, limit=None) + assert isinstance(res, ThreeDRevealNodeList) + assert mock_list_reveal_nodes_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + @pytest.fixture + def mock_list_reveal_sectors_response(self, rsps): + res = { + "data": { + "items": [ + { + "id": 1000, + "parentId": 900, + "path": "0/100/500/900/1000", + "depth": 4, + "boundingBox": {"max": [0, 0, 0], "min": [0, 0, 0]}, + "threedFiles": [{"version": 1, "fileId": 1000}], + } + ] + } + } + rsps.add( + rsps.GET, + re.compile(re.escape(THREE_D_API._base_url) + "/3d/reveal/models/1/revisions/1/sectors"), + status=200, + json=res, + ) + yield rsps + + def test_list_sectors(self, mock_list_reveal_sectors_response): + res = THREE_D_API.reveal.list_sectors( + model_id=1, revision_id=1, bounding_box={"max": [1, 1, 1], "min": [0, 0, 0]}, limit=None + ) + assert isinstance(res, ThreeDRevealSectorList) + assert "boundingBox=%7B" in mock_list_reveal_sectors_response.calls[0].request.url + assert mock_list_reveal_sectors_response.calls[0].response.json()["items"] == res.dump(camel_case=True) diff --git a/tests/tests_unit/test_api/test_assets.py b/tests/tests_unit/test_api/test_assets.py new file mode 100644 index 0000000000..e22235da17 --- /dev/null +++ b/tests/tests_unit/test_api/test_assets.py @@ -0,0 +1,438 @@ +import json +import queue +import re +import time +from collections import OrderedDict + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.assets import Asset, AssetList, AssetUpdate, _AssetPoster, _AssetPosterWorker +from cognite.client.data_classes import AssetFilter +from cognite.client.exceptions import CogniteAPIError +from tests.utils import jsgz_load, profilectx, set_request_limit + +COGNITE_CLIENT = CogniteClient() +ASSETS_API = COGNITE_CLIENT.assets + + +@pytest.fixture +def mock_assets_response(rsps): + response_body = { + "items": [ + { + "path": [0], + "externalId": "string", + "name": "string", + "parentId": 1, + "description": "string", + "metadata": {"metadata-key": "metadata-value"}, + "source": "string", + "id": 1, + "lastUpdatedTime": 0, + "depth": 0, + } + ] + } + + url_pattern = re.compile(re.escape(ASSETS_API._base_url) + "/.+") + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_get_subtree(rsps): + rsps.add( + rsps.POST, + ASSETS_API._base_url + "/assets/byids", + status=200, + json={"items": [{"id": 1, "path": [1], "depth": 0}]}, + ) + rsps.add( + rsps.POST, + ASSETS_API._base_url + "/assets/list", + status=200, + json={ + "items": [ + {"id": 2, "path": [1, 2], "depth": 1}, + {"id": 3, "path": [1, 3], "depth": 1}, + {"id": 4, "path": [1, 4], "depth": 1}, + ] + }, + ) + rsps.add( + rsps.POST, + ASSETS_API._base_url + "/assets/list", + status=200, + json={"items": [{"id": 5, "path": [1, 2, 5], "depth": 2}, {"id": 6, "path": [1, 2, 5], "depth": 2}]}, + ) + rsps.add( + rsps.POST, + ASSETS_API._base_url + "/assets/list", + status=200, + json={"items": [{"id": 7, "path": [1, 3, 7], "depth": 2}, {"id": 8, "path": [1, 3, 8], "depth": 2}]}, + ) + rsps.add( + rsps.POST, + ASSETS_API._base_url + "/assets/list", + status=200, + json={"items": [{"id": 9, "path": [1, 4, 9], "depth": 2}, {"id": 10, "path": [1, 4, 10], "depth": 2}]}, + ) + rsps.add(rsps.POST, ASSETS_API._base_url + "/assets/list", status=200, json={"items": []}) + yield rsps + + +class TestAssets: + def test_retrieve_single(self, mock_assets_response): + res = ASSETS_API.retrieve(id=1) + assert isinstance(res, Asset) + assert mock_assets_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_retrieve_multiple(self, mock_assets_response): + res = ASSETS_API.retrieve_multiple(ids=[1]) + assert isinstance(res, AssetList) + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list(self, mock_assets_response): + res = ASSETS_API.list(name="bla") + assert "bla" == jsgz_load(mock_assets_response.calls[0].request.body)["filter"]["name"] + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create_single(self, mock_assets_response): + res = ASSETS_API.create(Asset(external_id="1", name="blabla")) + assert isinstance(res, Asset) + assert mock_assets_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_assets_response): + res = ASSETS_API.create([Asset(external_id="1", name="blabla")]) + assert isinstance(res, AssetList) + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_iter_single(self, mock_assets_response): + for asset in ASSETS_API: + assert mock_assets_response.calls[0].response.json()["items"][0] == asset.dump(camel_case=True) + + def test_iter_chunk(self, mock_assets_response): + for assets in ASSETS_API(chunk_size=1): + assert mock_assets_response.calls[0].response.json()["items"] == assets.dump(camel_case=True) + + def test_delete_single(self, mock_assets_response): + res = ASSETS_API.delete(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_assets_response.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_assets_response): + res = ASSETS_API.delete(id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_assets_response.calls[0].request.body) + assert res is None + + def test_update_with_resource_class(self, mock_assets_response): + res = ASSETS_API.update(Asset(id=1)) + assert isinstance(res, Asset) + assert mock_assets_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_with_update_class(self, mock_assets_response): + res = ASSETS_API.update(AssetUpdate(id=1).description.set("blabla")) + assert isinstance(res, Asset) + assert mock_assets_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_multiple(self, mock_assets_response): + res = ASSETS_API.update([AssetUpdate(id=1).description.set("blabla")]) + assert isinstance(res, AssetList) + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_search(self, mock_assets_response): + res = ASSETS_API.search(filter=AssetFilter(name="1")) + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"search": {"name": None, "description": None}, "filter": {"name": "1"}, "limit": None} == jsgz_load( + mock_assets_response.calls[0].request.body + ) + + @pytest.mark.parametrize("filter_field", ["parent_ids", "parentIds"]) + def test_search_dict_filter(self, mock_assets_response, filter_field): + res = ASSETS_API.search(filter={filter_field: "bla"}) + assert mock_assets_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert { + "search": {"name": None, "description": None}, + "filter": {"parentIds": "bla"}, + "limit": None, + } == jsgz_load(mock_assets_response.calls[0].request.body) + + def test_get_subtree(self, mock_get_subtree): + assets = COGNITE_CLIENT.assets.retrieve_subtree(id=1) + assert len(assets) == 10 + for i, asset in enumerate(assets): + assert asset.id == i + 1 + + def test_get_subtree_w_depth(self, mock_get_subtree): + mock_get_subtree.assert_all_requests_are_fired = False + assets = COGNITE_CLIENT.assets.retrieve_subtree(id=1, depth=1) + assert len(assets) == 4 + for i, asset in enumerate(assets): + assert asset.id == i + 1 + + def test_assets_update_object(self): + assert isinstance( + AssetUpdate(1) + .description.set("") + .description.set(None) + .external_id.set("1") + .external_id.set(None) + .metadata.add({}) + .metadata.remove([]) + .metadata.set({}) + .metadata.set(None) + .name.set("") + .name.set(None) + .source.set(1) + .source.set(None), + AssetUpdate, + ) + + +class TestAssetPosterWorker: + def test_run(self, mock_assets_response): + q_req = queue.Queue() + q_res = queue.Queue() + + w = _AssetPosterWorker(request_queue=q_req, response_queue=q_res, client=ASSETS_API) + w.start() + q_req.put([Asset()]) + time.sleep(0.1) + w.stop = True + assert [Asset._load(mock_assets_response.calls[0].response.json()["items"][0])] == q_res.get() + assert 1 == len(mock_assets_response.calls) + + +def generate_asset_tree(root_external_id: str, depth: int, children_per_node: int, current_depth=1): + assert 1 <= children_per_node <= 10, "children_per_node must be between 1 and 10" + assets = [] + if current_depth == 1: + assets = [Asset(external_id=root_external_id)] + if depth > current_depth: + for i in range(children_per_node): + asset = Asset(parent_external_id=root_external_id, external_id="{}{}".format(root_external_id, i)) + assets.append(asset) + if depth > current_depth + 1: + assets.extend( + generate_asset_tree(root_external_id + str(i), depth, children_per_node, current_depth + 1) + ) + return assets + + +class TestAssetPoster: + def test_validate_asset_hierarchy_parent_ref_null_pointer(self): + assets = [Asset(parent_external_id="1", external_id="2")] + with pytest.raises(AssertionError, match="does not point"): + _AssetPoster(assets, ASSETS_API) + + def test_validate_asset_hierarchy_asset_has_parent_id_and_parent_ref_id(self): + assets = [Asset(external_id="1"), Asset(parent_external_id="1", parent_id=1, external_id="2")] + with pytest.raises(AssertionError, match="has both"): + _AssetPoster(assets, ASSETS_API) + + def test_validate_asset_hierarchy_duplicate_ref_ids(self): + assets = [Asset(external_id="1"), Asset(parent_external_id="1", external_id="1")] + with pytest.raises(AssertionError, match="Duplicate"): + _AssetPoster(assets, ASSETS_API) + + def test_validate_asset_hierarchy__more_than_limit_only_resolved_assets(self): + with set_request_limit(ASSETS_API, 1): + _AssetPoster([Asset(parent_id=1), Asset(parent_id=2)], ASSETS_API) + + def test_validate_asset_hierarchy_circular_dependencies(self): + assets = [ + Asset(external_id="1", parent_external_id="3"), + Asset(external_id="2", parent_external_id="1"), + Asset(external_id="3", parent_external_id="2"), + ] + with set_request_limit(ASSETS_API, 1): + with pytest.raises(AssertionError, match="circular dependencies"): + _AssetPoster(assets, ASSETS_API) + + def test_validate_asset_hierarchy_self_dependency(self): + assets = [Asset(external_id="1"), Asset(external_id="2", parent_external_id="2")] + with set_request_limit(ASSETS_API, 1): + with pytest.raises(AssertionError, match="circular dependencies"): + _AssetPoster(assets, ASSETS_API) + + def test_initialize(self): + assets = [ + Asset(external_id="1"), + Asset(external_id="3", parent_external_id="1"), + Asset(external_id="2", parent_external_id="1"), + Asset(external_id="4", parent_external_id="2"), + ] + + ap = _AssetPoster(assets, ASSETS_API) + assert OrderedDict({str(i): None for i in range(1, 5)}) == ap.remaining_external_ids + assert { + "1": {Asset(external_id="2", parent_external_id="1"), Asset(external_id="3", parent_external_id="1")}, + "2": {Asset(external_id="4", parent_external_id="2")}, + "3": set(), + "4": set(), + } == ap.external_id_to_children + assert {"1": 3, "2": 1, "3": 0, "4": 0} == ap.external_id_to_descendent_count + assert ap.assets_remaining() is True + assert 0 == len(ap.posted_assets) + assert ap.request_queue.empty() + assert ap.response_queue.empty() + assert {"1", "2", "3", "4"} == ap.remaining_external_ids_set + + def test_get_unblocked_assets__assets_unblocked_by_default_less_than_limit(self): + assets = generate_asset_tree(root_external_id="0", depth=4, children_per_node=10) + ap = _AssetPoster(assets=assets, client=ASSETS_API) + unblocked_assets_lists = ap._get_unblocked_assets() + assert 1 == len(unblocked_assets_lists) + assert 1000 == len(unblocked_assets_lists[0]) + + def test_get_unblocked_assets__assets_unblocked_by_default_more_than_limit(self): + assets = [] + for i in range(4): + assets.extend(generate_asset_tree(root_external_id=str(i), depth=2, children_per_node=2)) + with set_request_limit(ASSETS_API, 3): + ap = _AssetPoster(assets=assets, client=ASSETS_API) + unblocked_assets_lists = ap._get_unblocked_assets() + assert 4 == len(unblocked_assets_lists) + for li in unblocked_assets_lists: + assert 3 == len(li) + + @pytest.fixture + def mock_post_asset_hierarchy(self, rsps): + ASSETS_API._max_workers = 1 + + def request_callback(request): + items = jsgz_load(request.body)["items"] + response_assets = [] + for item in items: + parent_id = None + if "parentId" in item: + parent_id = item["parentId"] + if "parentExternalId" in item: + parent_id = item["parentExternalId"] + "id" + id = item.get("externalId", "root_") + "id" + response_assets.append( + { + "id": id, + "parentId": parent_id, + "externalId": item["externalId"], + "parentExternalId": item.get("parentExternalId"), + "path": [parent_id or "", id], + } + ) + return 200, {}, json.dumps({"items": response_assets}) + + rsps.add_callback( + rsps.POST, ASSETS_API._base_url + "/assets", callback=request_callback, content_type="application/json" + ) + yield rsps + ASSETS_API._max_workers = 10 + + @pytest.mark.parametrize( + "limit, depth, children_per_node, expected_num_calls", + [(100, 4, 10, 13), (9, 3, 9, 11), (100, 101, 1, 2), (1, 10, 1, 10)], + ) + def test_post_hierarchy(self, limit, depth, children_per_node, expected_num_calls, mock_post_asset_hierarchy): + assets = generate_asset_tree(root_external_id="0", depth=depth, children_per_node=children_per_node) + with set_request_limit(ASSETS_API, limit): + created_assets = ASSETS_API.create(assets) + + assert len(assets) == len(created_assets) + assert expected_num_calls - 1 <= len(mock_post_asset_hierarchy.calls) <= expected_num_calls + 1 + for asset in created_assets: + if asset.id == "0id": + assert asset.parent_id is None + else: + assert asset.id[:-3] == asset.parent_id[:-2] + + def test_post_assets_over_limit_only_resolved(self, mock_post_asset_hierarchy): + with set_request_limit(ASSETS_API, 1): + _AssetPoster([Asset(parent_id=1), Asset(parent_id=2)], ASSETS_API).post() + assert 2 == len(mock_post_asset_hierarchy.calls) + + @pytest.fixture + def mock_post_asset_hierarchy_with_failures(self, rsps): + def request_callback(request): + items = jsgz_load(request.body)["items"] + response_assets = [] + item = items[0] + parent_id = None + if "parentId" in item: + parent_id = item["parentId"] + if "parentExternalId" in item: + parent_id = item["parentExternalId"] + "id" + id = item.get("refId", "root_") + "id" + response_assets.append( + { + "id": id, + "parentId": parent_id, + "externalId": item["externalId"], + "parentExternalId": item.get("parentExternalId"), + "path": [parent_id or "", id], + } + ) + + if item["name"] == "400": + return 400, {}, json.dumps({"error": {"message": "user error", "code": 400}}) + + if item["name"] == "500": + return 500, {}, json.dumps({"error": {"message": "internal server error", "code": 500}}) + + return 200, {}, json.dumps({"items": response_assets}) + + rsps.add_callback( + rsps.POST, ASSETS_API._base_url + "/assets", callback=request_callback, content_type="application/json" + ) + with set_request_limit(ASSETS_API, 1): + yield rsps + + def test_post_with_failures(self, mock_post_asset_hierarchy_with_failures): + assets = [ + Asset(name="200", external_id="0"), + Asset(name="200", parent_external_id="0", external_id="01"), + Asset(name="400", parent_external_id="0", external_id="02"), + Asset(name="200", parent_external_id="02", external_id="021"), + Asset(name="200", parent_external_id="021", external_id="0211"), + Asset(name="500", parent_external_id="0", external_id="03"), + Asset(name="200", parent_external_id="03", external_id="031"), + ] + with pytest.raises(CogniteAPIError) as e: + ASSETS_API.create(assets) + + assert {a.external_id for a in e.value.unknown} == {"03"} + assert {a.external_id for a in e.value.failed} == {"02", "021", "0211", "031"} + assert {a.external_id for a in e.value.successful} == {"0", "01"} + + +@pytest.fixture +def mock_assets_empty(rsps): + url_pattern = re.compile(re.escape(ASSETS_API._base_url) + "/.+") + rsps.add(rsps.POST, url_pattern, status=200, json={"items": []}) + yield rsps + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_asset_list_to_pandas(self, mock_assets_response): + import pandas as pd + + df = ASSETS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert 1 == df.shape[0] + assert {"metadata-key": "metadata-value"} == df["metadata"][0] + + def test_asset_list_to_pandas_empty(self, mock_assets_empty): + import pandas as pd + + df = ASSETS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert df.empty + + def test_asset_to_pandas(self, mock_assets_response): + import pandas as pd + + df = ASSETS_API.retrieve(id=1).to_pandas() + assert isinstance(df, pd.DataFrame) + assert "metadata" not in df.columns + assert [0] == df.loc["path"][0] + assert "metadata-value" == df.loc["metadata-key"][0] diff --git a/tests/tests_unit/test_api/test_datapoints.py b/tests/tests_unit/test_api/test_datapoints.py new file mode 100644 index 0000000000..b0ce50920d --- /dev/null +++ b/tests/tests_unit/test_api/test_datapoints.py @@ -0,0 +1,949 @@ +import json +import math +from contextlib import contextmanager +from datetime import datetime +from random import choice, random +from unittest import mock +from unittest.mock import PropertyMock + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.datapoints import _DatapointsFetcher, _DPQuery, _DPWindow +from cognite.client.data_classes import Datapoint, Datapoints, DatapointsList, DatapointsQuery +from cognite.client.exceptions import CogniteAPIError +from cognite.client.utils import _utils as utils +from tests.utils import jsgz_load, set_request_limit + +COGNITE_CLIENT = CogniteClient() +DPS_CLIENT = COGNITE_CLIENT.datapoints + + +def generate_datapoints(start: int, end: int, aggregates=None, granularity=None): + dps = [] + granularity = utils.granularity_to_ms(granularity) if granularity else 1000 + for i in range(start, end, granularity): + dp = {} + if aggregates: + if aggregates == ["count"]: + dp["count"] = int(math.ceil((end - start) / 1000)) + else: + for agg in aggregates: + dp[agg] = random() + else: + dp["value"] = random() + dp["timestamp"] = i + dps.append(dp) + return dps + + +@pytest.fixture +def mock_get_datapoints(rsps): + def request_callback(request): + payload = jsgz_load(request.body) + + items = [] + for dps_query in payload["items"]: + aggregates = [] + + if "aggregates" in dps_query: + aggregates = dps_query["aggregates"] + elif "aggregates" in payload: + aggregates = payload["aggregates"] + + granularity = None + if "granularity" in dps_query: + granularity = dps_query["granularity"] + elif "granularity" in payload: + granularity = payload["granularity"] + + if (granularity and not aggregates) or (not granularity and aggregates): + return ( + 400, + {}, + json.dumps({"error": {"code": 400, "message": "You must specify both aggregates AND granularity"}}), + ) + + if "start" in dps_query and "end" in dps_query: + start, end = dps_query["start"], dps_query["end"] + else: + start, end = payload["start"], payload["end"] + + limit = 100000 + if "limit" in dps_query: + limit = dps_query["limit"] + elif "limit" in payload: + limit = payload["limit"] + + dps = generate_datapoints(start, end, aggregates, granularity) + dps = dps[:limit] + id_to_return = dps_query.get("id", int(dps_query.get("externalId", "-1"))) + external_id_to_return = dps_query.get("externalId", str(dps_query.get("id", -1))) + items.append({"id": id_to_return, "externalId": external_id_to_return, "datapoints": dps}) + response = {"items": items} + return 200, {}, json.dumps(response) + + rsps.add_callback( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + callback=request_callback, + content_type="application/json", + ) + yield rsps + + +@pytest.fixture +def mock_get_datapoints_empty(rsps): + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + status=200, + json={"items": [{"id": 1, "externalId": "1", "datapoints": []}]}, + ) + yield rsps + + +@pytest.fixture +def mock_get_datapoints_one_ts_empty(rsps): + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + status=200, + json={"items": [{"id": 1, "externalId": "1", "datapoints": [{"timestamp": 1, "value": 1}]}]}, + ) + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + status=200, + json={"items": [{"id": 2, "externalId": "2", "datapoints": []}]}, + ) + yield rsps + + +@pytest.fixture +def mock_get_datapoints_one_ts_has_missing_aggregates(rsps): + def callback(request): + item = jsgz_load(request.body)["items"][0] + if item["aggregates"] == ["average"]: + dps = { + "id": 1, + "externalId": "abc", + "datapoints": [ + {"timestamp": 0, "average": 0}, + {"timestamp": 1, "average": 1}, + {"timestamp": 2, "average": 2}, + {"timestamp": 3, "average": 3}, + {"timestamp": 4, "average": 4}, + ], + } + else: + dps = { + "id": 2, + "externalId": "def", + "datapoints": [ + {"timestamp": 0}, + {"timestamp": 1, "interpolation": 1}, + {"timestamp": 2}, + {"timestamp": 3, "interpolation": 3}, + {"timestamp": 4}, + ], + } + return 200, {}, json.dumps({"items": [dps]}) + + rsps.add_callback( + rsps.POST, DPS_CLIENT._base_url + "/timeseries/data/list", callback=callback, content_type="application/json" + ) + yield rsps + + +@pytest.fixture +def set_dps_workers(): + def set_workers(limit): + DPS_CLIENT._max_workers = limit + + workers_tmp = DPS_CLIENT._max_workers + yield set_workers + DPS_CLIENT._max_workers = workers_tmp + + +def assert_dps_response_is_correct(calls, dps_object): + datapoints = [] + for call in calls: + if jsgz_load(call.request.body)["limit"] > 1 and jsgz_load(call.request.body).get("aggregates") != ["count"]: + dps_response = call.response.json()["items"][0] + if dps_response["id"] == dps_object.id and dps_response["externalId"] == dps_object.external_id: + datapoints.extend(dps_response["datapoints"]) + id = dps_response["id"] + external_id = dps_response["externalId"] + + expected_dps = sorted(datapoints, key=lambda x: x["timestamp"]) + assert id == dps_object.id + assert external_id == dps_object.external_id + assert expected_dps == dps_object.dump(camel_case=True)["datapoints"] + + +class TestGetDatapoints: + def test_retrieve_datapoints_by_id(self, mock_get_datapoints): + dps_res = DPS_CLIENT.retrieve(id=123, start=1000000, end=1100000) + assert isinstance(dps_res, Datapoints) + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_retrieve_datapoints_500(self, rsps): + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + json={"error": {"code": 500, "message": "Internal Server Error"}}, + status=500, + ) + with pytest.raises(CogniteAPIError): + DPS_CLIENT.retrieve(id=123, start=1000000, end=1100000) + + def test_retrieve_datapoints_by_external_id(self, mock_get_datapoints): + dps_res = DPS_CLIENT.retrieve(external_id="123", start=1000000, end=1100000) + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_retrieve_datapoints_aggregates(self, mock_get_datapoints): + dps_res = DPS_CLIENT.retrieve( + id=123, start=1000000, end=1100000, aggregates=["average", "stepInterpolation"], granularity="10s" + ) + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_retrieve_datapoints_local_aggregates(self, mock_get_datapoints): + dps_res_list = DPS_CLIENT.retrieve( + external_id={"externalId": "123", "aggregates": ["average"]}, + id={"id": 234}, + start=1000000, + end=1100000, + aggregates=["max"], + granularity="10s", + ) + for dps_res in dps_res_list: + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_retrieve_datapoints_some_aggregates_omitted(self, mock_get_datapoints_one_ts_has_missing_aggregates): + dps_res_list = DPS_CLIENT.retrieve( + id={"id": 1, "aggregates": ["average"]}, + external_id={"externalId": "def", "aggregates": ["interpolation"]}, + start=0, + end=1, + aggregates=[], + granularity="1s", + ) + for dps in dps_res_list: + if dps.id == 1: + assert dps.average == [0, 1, 2, 3, 4] + elif dps.id == 2: + assert dps.interpolation == [None, 1, None, 3, None] + + def test_datapoints_paging(self, mock_get_datapoints, set_dps_workers): + set_dps_workers(1) + with set_request_limit(DPS_CLIENT, 2): + dps_res = DPS_CLIENT.retrieve(id=123, start=0, end=10000, aggregates=["average"], granularity="1s") + assert 6 == len(mock_get_datapoints.calls) + assert 10 == len(dps_res) + + def test_datapoints_concurrent(self, mock_get_datapoints): + DPS_CLIENT._DPS_LIMIT_AGG = 20 + dps_res = DPS_CLIENT.retrieve(id=123, start=0, end=100000, aggregates=["average"], granularity="1s") + requested_windows = sorted( + [ + (jsgz_load(call.request.body)["start"], jsgz_load(call.request.body)["end"]) + for call in mock_get_datapoints.calls + ], + key=lambda x: x[0], + ) + assert (0, 100000) == requested_windows[0] + assert [(20000, 100000), (40000, 100000), (60000, 100000), (80000, 100000)] == requested_windows[2:] + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_datapoints_paging_with_limit(self, mock_get_datapoints): + with set_request_limit(DPS_CLIENT, 3): + dps_res = DPS_CLIENT.retrieve(id=123, start=0, end=10000, aggregates=["average"], granularity="1s", limit=4) + assert 4 == len(dps_res) + + def test_retrieve_datapoints_multiple_time_series(self, mock_get_datapoints): + ids = [1, 2, 3] + external_ids = ["4", "5", "6"] + dps_res_list = DPS_CLIENT.retrieve(id=ids, external_id=external_ids, start=0, end=100000) + assert isinstance(dps_res_list, DatapointsList), type(dps_res_list) + for dps_res in dps_res_list: + if dps_res.id in ids: + ids.remove(dps_res.id) + if dps_res.external_id in external_ids: + external_ids.remove(dps_res.external_id) + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + assert 0 == len(ids) + assert 0 == len(external_ids) + + def test_retrieve_datapoints_empty(self, mock_get_datapoints_empty): + res = DPS_CLIENT.retrieve(id=1, start=0, end=10000) + assert 0 == len(res) + + +class TestQueryDatapoints: + def test_query_single(self, mock_get_datapoints): + dps_res = DPS_CLIENT.query(query=DatapointsQuery(id=1, start=0, end=10000)) + assert isinstance(dps_res, Datapoints) + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_query_multiple(self, mock_get_datapoints): + dps_res_list = DPS_CLIENT.query( + query=[ + DatapointsQuery(id=1, start=0, end=10000), + DatapointsQuery(external_id="2", start=10000, end=20000, aggregates=["average"], granularity="2s"), + ] + ) + assert isinstance(dps_res_list, DatapointsList) + for dps_res in dps_res_list: + assert_dps_response_is_correct(mock_get_datapoints.calls, dps_res) + + def test_query_empty(self, mock_get_datapoints_empty): + dps_res = DPS_CLIENT.query(query=DatapointsQuery(id=1, start=0, end=10000)) + assert 0 == len(dps_res) + + +@pytest.fixture +def mock_retrieve_latest(rsps): + def request_callback(request): + payload = jsgz_load(request.body) + + items = [] + for latest_query in payload["items"]: + id = latest_query.get("id", -1) + external_id = latest_query.get("externalId", "-1") + before = latest_query.get("before", 10001) + items.append( + {"id": id, "externalId": external_id, "datapoints": [{"timestamp": before - 1, "value": random()}]} + ) + return 200, {}, json.dumps({"items": items}) + + rsps.add_callback( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/latest", + callback=request_callback, + content_type="application/json", + ) + yield rsps + + +@pytest.fixture +def mock_retrieve_latest_empty(rsps): + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/latest", + status=200, + json={ + "items": [{"id": 1, "externalId": "1", "datapoints": []}, {"id": 2, "externalId": "2", "datapoints": []}] + }, + ) + yield rsps + + +class TestGetLatest: + def test_retrieve_latest(self, mock_retrieve_latest): + res = DPS_CLIENT.retrieve_latest(id=1) + assert isinstance(res, Datapoints) + assert 10000 == res[0].timestamp + assert isinstance(res[0].value, float) + + def test_retrieve_latest_multiple_ts(self, mock_retrieve_latest): + res = DPS_CLIENT.retrieve_latest(id=1, external_id="2") + assert isinstance(res, DatapointsList) + for dps in res: + assert 10000 == dps[0].timestamp + assert isinstance(dps[0].value, float) + + def test_retrieve_latest_with_before(self, mock_retrieve_latest): + res = DPS_CLIENT.retrieve_latest(id=1, before=10) + assert isinstance(res, Datapoints) + assert 9 == res[0].timestamp + assert isinstance(res[0].value, float) + + def test_retrieve_latest_multiple_ts_with_before(self, mock_retrieve_latest): + res = DPS_CLIENT.retrieve_latest(id=[1, 2], external_id=["1", "2"], before=10) + assert isinstance(res, DatapointsList) + for dps in res: + assert 9 == dps[0].timestamp + assert isinstance(dps[0].value, float) + + def test_retrieve_latest_empty(self, mock_retrieve_latest_empty): + res = DPS_CLIENT.retrieve_latest(id=1) + assert isinstance(res, Datapoints) + assert 0 == len(res) + + def test_retrieve_latest_multiple_ts_empty(self, mock_retrieve_latest_empty): + res_list = DPS_CLIENT.retrieve_latest(id=[1, 2]) + assert isinstance(res_list, DatapointsList) + assert 2 == len(res_list) + for res in res_list: + assert 0 == len(res) + + +@pytest.fixture +def mock_post_datapoints(rsps): + rsps.add(rsps.POST, DPS_CLIENT._base_url + "/timeseries/data", status=200, json={}) + yield rsps + + +class TestInsertDatapoints: + def test_insert_tuples(self, mock_post_datapoints): + dps = [(i * 1e11, i) for i in range(1, 11)] + res = DPS_CLIENT.insert(dps, id=1) + assert res is None + assert { + "items": [{"id": 1, "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 11)]}] + } == jsgz_load(mock_post_datapoints.calls[0].request.body) + + def test_insert_dicts(self, mock_post_datapoints): + dps = [{"timestamp": i * 1e11, "value": i} for i in range(1, 11)] + res = DPS_CLIENT.insert(dps, id=1) + assert res is None + assert { + "items": [{"id": 1, "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 11)]}] + } == jsgz_load(mock_post_datapoints.calls[0].request.body) + + def test_by_external_id(self, mock_post_datapoints): + dps = [(i * 1e11, i) for i in range(1, 11)] + DPS_CLIENT.insert(dps, external_id="1") + assert { + "items": [ + {"externalId": "1", "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 11)]} + ] + } == jsgz_load(mock_post_datapoints.calls[0].request.body) + + def test_insert_datapoints_in_jan_1970(self): + dps = [{"timestamp": i, "value": i} for i in range(1, 11)] + with pytest.raises(AssertionError): + DPS_CLIENT.insert(dps, id=1) + + @pytest.mark.parametrize("ts_key, value_key", [("timestamp", "values"), ("timstamp", "value")]) + def test_invalid_datapoints_keys(self, ts_key, value_key): + dps = [{ts_key: i * 1e11, value_key: i} for i in range(1, 11)] + with pytest.raises(AssertionError, match="is missing the"): + DPS_CLIENT.insert(dps, id=1) + + def test_insert_datapoints_over_limit(self, mock_post_datapoints): + dps = [(i * 1e11, i) for i in range(1, 11)] + with set_request_limit(DPS_CLIENT, 5): + res = DPS_CLIENT.insert(dps, id=1) + assert res is None + request_bodies = [jsgz_load(call.request.body) for call in mock_post_datapoints.calls] + assert { + "items": [{"id": 1, "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 6)]}] + } in request_bodies + assert { + "items": [{"id": 1, "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(6, 11)]}] + } in request_bodies + + def test_insert_datapoints_no_data(self): + with pytest.raises(AssertionError, match="No datapoints provided"): + DPS_CLIENT.insert(id=1, datapoints=[]) + + def test_insert_datapoints_in_multiple_time_series(self, mock_post_datapoints): + dps = [{"timestamp": i * 1e11, "value": i} for i in range(1, 11)] + dps_objects = [{"externalId": "1", "datapoints": dps}, {"id": 1, "datapoints": dps}] + res = DPS_CLIENT.insert_multiple(dps_objects) + assert res is None + request_bodies = [jsgz_load(call.request.body) for call in mock_post_datapoints.calls] + assert { + "items": [{"id": 1, "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 11)]}] + } in request_bodies + assert { + "items": [ + {"externalId": "1", "datapoints": [{"timestamp": int(i * 1e11), "value": i} for i in range(1, 11)]} + ] + } in request_bodies + + def test_insert_datapoints_in_multiple_time_series_invalid_key(self): + dps = [{"timestamp": i * 1e11, "value": i} for i in range(1, 11)] + dps_objects = [{"extId": "1", "datapoints": dps}] + with pytest.raises(AssertionError, match="Invalid key 'extId'"): + DPS_CLIENT.insert_multiple(dps_objects) + + +@pytest.fixture +def mock_delete_datapoints(rsps): + rsps.add(rsps.POST, DPS_CLIENT._base_url + "/timeseries/data/delete", status=200, json={}) + yield rsps + + +class TestDeleteDatapoints: + def test_delete_range(self, mock_delete_datapoints): + res = DPS_CLIENT.delete_range(start=datetime(2018, 1, 1), end=datetime(2018, 1, 2), id=1) + assert res is None + assert {"items": [{"id": 1, "inclusiveBegin": 1514764800000, "exclusiveEnd": 1514851200000}]} == jsgz_load( + mock_delete_datapoints.calls[0].request.body + ) + + @pytest.mark.parametrize( + "id, external_id, exception", + [(None, None, AssertionError), (1, "1", AssertionError), ("1", None, TypeError), (None, 1, TypeError)], + ) + def test_delete_range_invalid_id(self, id, external_id, exception): + with pytest.raises(exception): + DPS_CLIENT.delete_range("1d-ago", "now", id, external_id) + + def test_delete_range_start_after_end(self): + with pytest.raises(AssertionError, match="must be"): + DPS_CLIENT.delete_range(1, 0, 1) + + def test_delete_ranges(self, mock_delete_datapoints): + ranges = [{"id": 1, "start": 0, "end": 1}, {"externalId": "1", "start": 0, "end": 1}] + DPS_CLIENT.delete_ranges(ranges) + assert { + "items": [ + {"id": 1, "inclusiveBegin": 0, "exclusiveEnd": 1}, + {"externalId": "1", "inclusiveBegin": 0, "exclusiveEnd": 1}, + ] + } == jsgz_load(mock_delete_datapoints.calls[0].request.body) + + def test_delete_ranges_invalid_ids(self): + ranges = [{"idz": 1, "start": 0, "end": 1}] + with pytest.raises(AssertionError, match="Invalid key 'idz'"): + DPS_CLIENT.delete_ranges(ranges) + ranges = [{"start": 0, "end": 1}] + with pytest.raises(AssertionError, match="Exactly one of id and external id must be specified"): + DPS_CLIENT.delete_ranges(ranges) + + +class TestDatapointsObject: + def test_len(self): + assert 3 == len(Datapoints(id=1, timestamp=[1, 2, 3], value=[1, 2, 3])) + + def test_get_non_empty_data_fields(self): + assert sorted([("timestamp", [1, 2, 3]), ("value", [1, 2, 3])]) == sorted( + Datapoints(id=1, timestamp=[1, 2, 3], value=[1, 2, 3])._get_non_empty_data_fields() + ) + assert sorted([("timestamp", [1, 2, 3]), ("max", [1, 2, 3]), ("sum", [1, 2, 3])]) == sorted( + Datapoints(id=1, timestamp=[1, 2, 3], sum=[1, 2, 3], max=[1, 2, 3])._get_non_empty_data_fields() + ) + assert sorted([("timestamp", [1, 2, 3]), ("max", [1, 2, 3])]) == sorted( + Datapoints(id=1, timestamp=[1, 2, 3], sum=[], max=[1, 2, 3])._get_non_empty_data_fields() + ) + assert sorted([("timestamp", [1, 2, 3]), ("max", [1, 2, 3]), ("sum", [])]) == sorted( + Datapoints(id=1, timestamp=[1, 2, 3], sum=[], max=[1, 2, 3])._get_non_empty_data_fields( + get_empty_lists=True + ) + ) + assert [("timestamp", [])] == list(Datapoints(id=1)._get_non_empty_data_fields()) + + def test_iter(self): + for dp in Datapoints(id=1, timestamp=[1, 2, 3], value=[1, 2, 3]): + assert dp.timestamp in [1, 2, 3] + assert dp.value in [1, 2, 3] + + def test_eq(self): + assert Datapoints(1) == Datapoints(1) + assert Datapoints(1, timestamp=[1, 2, 3], value=[1, 2, 3]) == Datapoints( + 1, timestamp=[1, 2, 3], value=[1, 2, 3] + ) + assert Datapoints(1) != Datapoints(0) + assert Datapoints(1, timestamp=[1, 2, 3], value=[1, 2, 3]) != Datapoints(1, timestamp=[1, 2, 3], max=[1, 2, 3]) + assert Datapoints(1, timestamp=[1, 2, 3], value=[1, 2, 3]) != Datapoints( + 1, timestamp=[1, 2, 3], value=[1, 2, 4] + ) + + def test_get_item(self): + dps = Datapoints(id=1, timestamp=[1, 2, 3], value=[1, 2, 3]) + + assert Datapoint(timestamp=1, value=1) == dps[0] + assert Datapoint(timestamp=2, value=2) == dps[1] + assert Datapoint(timestamp=3, value=3) == dps[2] + assert Datapoints(id=1, timestamp=[1, 2], value=[1, 2]) == dps[:2] + + def test_load(self): + res = Datapoints._load( + {"id": 1, "externalId": "1", "datapoints": [{"timestamp": 1, "value": 1}, {"timestamp": 2, "value": 2}]} + ) + assert 1 == res.id + assert "1" == res.external_id + assert [1, 2] == res.timestamp + assert [1, 2] == res.value + + def test_slice(self): + res = Datapoints(id=1, timestamp=[1, 2, 3])._slice(slice(None, 1)) + assert [1] == res.timestamp + + def test_insert(self): + d0 = Datapoints() + d1 = Datapoints(id=1, external_id="1", timestamp=[7, 8, 9], value=[7, 8, 9]) + d2 = Datapoints(id=1, external_id="1", timestamp=[1, 2, 3], value=[1, 2, 3]) + d3 = Datapoints(id=1, external_id="1", timestamp=[4, 5, 6], value=[4, 5, 6]) + + d0._insert(d1) + assert [7, 8, 9] == d0.timestamp + assert [7, 8, 9] == d0.value + assert 1 == d0.id + assert "1" == d0.external_id + assert d0.sum == None + + d0._insert(d2) + assert [1, 2, 3, 7, 8, 9] == d0.timestamp + assert [1, 2, 3, 7, 8, 9] == d0.value + assert 1 == d0.id + assert "1" == d0.external_id + assert d0.sum == None + + d0._insert(d3) + assert [1, 2, 3, 4, 5, 6, 7, 8, 9] == d0.timestamp + assert [1, 2, 3, 4, 5, 6, 7, 8, 9] == d0.value + assert 1 == d0.id + assert "1" == d0.external_id + assert d0.sum == None + + +@pytest.mark.dsl +class TestPlotDatapoints: + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.plot") + def test_plot_datapoints(self, pandas_plot_mock, plt_show_mock): + d = Datapoints(id=1, timestamp=[1, 2, 3, 4, 5], value=[1, 2, 3, 4, 5]) + d.plot() + assert 1 == pandas_plot_mock.call_count + assert 1 == plt_show_mock.call_count + + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.plot") + def test_plot_datapoints_list(self, pandas_plot_mock, plt_show_mock): + d1 = Datapoints(id=1, timestamp=[1, 2, 3, 4, 5], value=[1, 2, 3, 4, 5]) + d2 = Datapoints(id=2, timestamp=[1, 2, 3, 4, 5], value=[6, 7, 8, 9, 10]) + d = DatapointsList([d1, d2]) + d.plot() + assert 1 == pandas_plot_mock.call_count + assert 1 == plt_show_mock.call_count + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_datapoint(self): + import pandas as pd + + d = Datapoint(timestamp=0, value=2, max=3) + expected_df = pd.DataFrame({"value": [2], "max": [3]}, index=[utils.ms_to_datetime(0)]) + pd.testing.assert_frame_equal(expected_df, d.to_pandas(), check_like=True) + + def test_datapoints(self): + import pandas as pd + + d = Datapoints(id=1, timestamp=[1, 2, 3], average=[2, 3, 4], step_interpolation=[3, 4, 5]) + expected_df = pd.DataFrame( + {"1|average": [2, 3, 4], "1|stepInterpolation": [3, 4, 5]}, + index=[utils.ms_to_datetime(ms) for ms in [1, 2, 3]], + ) + pd.testing.assert_frame_equal(expected_df, d.to_pandas()) + + def test_id_and_external_id_set_gives_external_id_columns(self): + import pandas as pd + + d = Datapoints(id=0, external_id="abc", timestamp=[1, 2, 3], average=[2, 3, 4], step_interpolation=[3, 4, 5]) + expected_df = pd.DataFrame( + {"abc|average": [2, 3, 4], "abc|stepInterpolation": [3, 4, 5]}, + index=[utils.ms_to_datetime(ms) for ms in [1, 2, 3]], + ) + pd.testing.assert_frame_equal(expected_df, d.to_pandas()) + + def test_datapoints_empty(self): + d = Datapoints(external_id="1", timestamp=[], value=[]) + assert d.to_pandas().empty + + def test_datapoints_list(self): + import pandas as pd + + d1 = Datapoints(id=1, timestamp=[1, 2, 3], average=[2, 3, 4], step_interpolation=[3, 4, 5]) + d2 = Datapoints(id=2, timestamp=[1, 2, 3], max=[2, 3, 4], step_interpolation=[3, 4, 5]) + d3 = Datapoints(id=3, timestamp=[1, 3], value=[1, 3]) + dps_list = DatapointsList([d1, d2, d3]) + expected_df = pd.DataFrame( + { + "1|average": [2, 3, 4], + "1|stepInterpolation": [3, 4, 5], + "2|max": [2, 3, 4], + "2|stepInterpolation": [3, 4, 5], + "3": [1, None, 3], + }, + index=[utils.ms_to_datetime(ms) for ms in [1, 2, 3]], + ) + pd.testing.assert_frame_equal(expected_df, dps_list.to_pandas()) + + def test_datapoints_list_non_aligned(self): + import pandas as pd + + d1 = Datapoints(id=1, timestamp=[1, 2, 3], value=[1, 2, 3]) + d2 = Datapoints(id=2, timestamp=[3, 4, 5], value=[3, 4, 5]) + + dps_list = DatapointsList([d1, d2]) + + expected_df = pd.DataFrame( + {"1": [1, 2, 3, None, None], "2": [None, None, 3, 4, 5]}, + index=[utils.ms_to_datetime(ms) for ms in [1, 2, 3, 4, 5]], + ) + pd.testing.assert_frame_equal(expected_df, dps_list.to_pandas()) + + def test_datapoints_list_empty(self): + dps_list = DatapointsList([]) + assert dps_list.to_pandas().empty + + def test_retrieve_dataframe(self, mock_get_datapoints): + df = DPS_CLIENT.retrieve_dataframe( + id=[1, {"id": 2, "aggregates": ["max"]}], + external_id=["123"], + start=1000000, + end=1100000, + aggregates=["average"], + granularity="10s", + ) + + assert {"1|average", "2|max", "123|average"} == set(df.columns) + assert df.shape[0] > 0 + + def test_retrieve_datapoints_some_aggregates_omitted(self, mock_get_datapoints_one_ts_has_missing_aggregates): + import pandas as pd + + df = DPS_CLIENT.retrieve_dataframe( + id={"id": 1, "aggregates": ["average"]}, + external_id={"externalId": "def", "aggregates": ["interpolation"]}, + start=0, + end=1, + aggregates=[], + granularity="1s", + ) + + expected_df = pd.DataFrame( + {"1|average": [0, 1, 2, 3, 4], "def|interpolation": [None, 1, None, 3, None]}, + index=[utils.ms_to_datetime(i) for i in range(5)], + ) + pd.testing.assert_frame_equal(df, expected_df) + + def test_retrieve_dataframe_id_and_external_id_requested(self, rsps): + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + status=200, + json={"items": [{"id": 1, "externalId": "abc", "datapoints": [{"timestamp": 0, "average": 1}]}]}, + ) + rsps.add( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + status=200, + json={"items": [{"id": 2, "externalId": "def", "datapoints": [{"timestamp": 0, "average": 1}]}]}, + ) + res = DPS_CLIENT.retrieve_dataframe( + start=0, end="now", id=1, external_id=["def"], aggregates=["average"], granularity="1m" + ) + assert {"1|average", "def|average"} == set(res.columns) + + def test_insert_dataframe(self, mock_post_datapoints): + import pandas as pd + + timestamps = [1500000000000, 1510000000000, 1520000000000, 1530000000000] + df = pd.DataFrame( + {"123": [1, 2, 3, 4], "456": [5.0, 6.0, 7.0, 8.0]}, index=[utils.ms_to_datetime(ms) for ms in timestamps] + ) + res = DPS_CLIENT.insert_dataframe(df) + assert res is None + request_bodies = [jsgz_load(call.request.body) for call in mock_post_datapoints.calls] + assert { + "items": [ + {"id": 123, "datapoints": [{"timestamp": ts, "value": val} for ts, val in zip(timestamps, range(1, 5))]} + ] + } in request_bodies + assert { + "items": [ + { + "id": 456, + "datapoints": [{"timestamp": ts, "value": float(val)} for ts, val in zip(timestamps, range(5, 9))], + } + ] + } in request_bodies + + def test_insert_dataframe_with_nans(self): + import pandas as pd + + timestamps = [1500000000000, 1510000000000, 1520000000000, 1530000000000] + df = pd.DataFrame( + {"123": [1, 2, None, 4], "456": [5.0, 6.0, 7.0, 8.0]}, index=[utils.ms_to_datetime(ms) for ms in timestamps] + ) + with pytest.raises(AssertionError, match="contains NaNs"): + DPS_CLIENT.insert_dataframe(df) + + def test_retrieve_datapoints_multiple_time_series_correct_ordering(self, mock_get_datapoints): + ids = [1, 2, 3] + external_ids = ["4", "5", "6"] + dps_res_list = DPS_CLIENT.retrieve(id=ids, external_id=external_ids, start=0, end=100000) + assert list(dps_res_list.to_pandas().columns) == ["1", "2", "3", "4", "5", "6"], "Incorrect column ordering" + + def test_retrieve_datapoints_one_ts_empty_correct_number_of_columns(self, mock_get_datapoints_one_ts_empty): + res = DPS_CLIENT.retrieve(id=[1, 2], start=0, end=10000) + assert 2 == len(res.to_pandas().columns) + + +@pytest.fixture +def mock_get_dps_count(rsps): + def request_callback(request): + payload = jsgz_load(request.body) + granularity = payload["granularity"] + aggregates = payload["aggregates"] + start = payload["start"] + end = payload["end"] + + assert payload["aggregates"] == ["count"] + assert utils.granularity_to_ms(payload["granularity"]) >= utils.granularity_to_ms("1d") + + dps = [{"timestamp": i, "count": 1000} for i in range(start, end, utils.granularity_to_ms(granularity))] + response = {"items": [{"id": 0, "externalId": "bla", "datapoints": dps}]} + return 200, {}, json.dumps(response) + + rsps.add_callback( + rsps.POST, + DPS_CLIENT._base_url + "/timeseries/data/list", + callback=request_callback, + content_type="application/json", + ) + yield rsps + + +gms = lambda s: utils.granularity_to_ms(s) + + +class TestDataFetcher: + @pytest.mark.parametrize( + "q, expected_q", + [ + ([_DPQuery(1, 2, None, None, None, None, None)], [_DPQuery(1, 2, None, None, None, None, None)]), + ( + [_DPQuery(datetime(2018, 1, 1), datetime(2019, 1, 1), None, None, None, None, None)], + [_DPQuery(1514764800000, 1546300800000, None, None, None, None, None)], + ), + ( + [_DPQuery(gms("1h"), gms(("25h")), None, ["average"], "1d", None, None)], + [_DPQuery(gms("1d"), gms("2d"), None, ["average"], "1d", None, None)], + ), + ], + ) + def test_preprocess_queries(self, q, expected_q): + _DatapointsFetcher(DPS_CLIENT)._preprocess_queries(q) + for actual, expected in zip(q, expected_q): + assert expected.start == actual.start + assert expected.end == actual.end + + @pytest.mark.parametrize( + "ts, granularity, expected_output", + [ + (gms("1h"), "1d", gms("1d")), + (gms("23h"), "10d", gms("1d")), + (gms("24h"), "5d", gms("1d")), + (gms("25h"), "3d", gms("2d")), + (gms("1m"), "1h", gms("1h")), + (gms("90s"), "10m", gms("2m")), + (gms("90s"), "1s", gms("90s")), + ], + ) + def test_align_with_granularity_unit(self, ts, granularity, expected_output): + assert expected_output == _DatapointsFetcher._align_with_granularity_unit(ts, granularity) + + @pytest.mark.parametrize( + "start, end, granularity, request_limit, user_limit, expected_output", + [ + (0, gms("20d"), "10d", 2, None, [_DPWindow(start=0, end=1728000000)]), + ( + 0, + gms("20d"), + "10d", + 1, + None, + [_DPWindow(start=0, end=864000000), _DPWindow(start=864000000, end=1728000000)], + ), + ( + 0, + gms("6d"), + "1s", + 2000, + None, + [_DPWindow(0, gms("2d")), _DPWindow(gms("2d"), gms("4d")), _DPWindow(gms("4d"), gms("6d"))], + ), + ( + 0, + gms("3d"), + None, + 1000, + None, + [_DPWindow(0, gms("1d")), _DPWindow(gms("1d"), gms("2d")), _DPWindow(gms("2d"), gms("3d"))], + ), + (0, gms("1h"), None, 2000, None, [_DPWindow(start=0, end=3600000)]), + (0, gms("1s"), None, 1, None, [_DPWindow(start=0, end=1000)]), + (0, gms("1s"), None, 1, None, [_DPWindow(start=0, end=1000)]), + (0, gms("3d"), None, 1000, 500, [_DPWindow(0, gms("1d"))]), + ], + ) + def test_get_datapoints_windows( + self, start, end, granularity, request_limit, user_limit, expected_output, mock_get_dps_count + ): + res = _DatapointsFetcher(DPS_CLIENT)._get_windows( + id=0, start=start, end=end, granularity=granularity, request_limit=request_limit, user_limit=user_limit + ) + assert expected_output == res + + @pytest.mark.parametrize( + "start, end, granularity, expected_output", + [ + (0, 10001, "1s", 10000), + (0, 10000, "1m", 0), + (0, 110000, "1m", gms("1m")), + (0, gms("10d") - 1, "2d", gms("8d")), + (0, gms("10d") - 1, "1m", gms("10d") - gms("1m")), + (0, 10000, "1s", 10000), + ], + ) + def test_align_window_end(self, start, end, granularity, expected_output): + assert expected_output == _DatapointsFetcher._align_window_end(start, end, granularity) + + def test_remove_duplicates_from_datapoints(self): + d = Datapoints( + id=1, + timestamp=[1, 1, 2, 3, 3, 4, 5, 5, 6, 7, 7], + value=[0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1], + max=[0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1], + ) + d_no_dupes = _DatapointsFetcher._remove_duplicates(d) + assert [1, 2, 3, 4, 5, 6, 7] == d_no_dupes.timestamp + assert [0, 1, 0, 1, 1, 0, 1] == d_no_dupes.value + assert [0, 1, 0, 1, 1, 0, 1] == d_no_dupes.max + + +class TestHelpers: + @pytest.mark.parametrize( + "ids, external_ids, expected_output", + [ + (1, None, ([{"id": 1}], True)), + (None, "1", ([{"externalId": "1"}], True)), + (1, "1", ([{"id": 1}, {"externalId": "1"}], False)), + ([1], ["1"], ([{"id": 1}, {"externalId": "1"}], False)), + ([1], None, ([{"id": 1}], False)), + ({"id": 1, "aggregates": ["average"]}, None, ([{"id": 1, "aggregates": ["average"]}], True)), + ({"id": 1}, {"externalId": "1"}, ([{"id": 1}, {"externalId": "1"}], False)), + ( + [{"id": 1, "aggregates": ["average"]}], + [{"externalId": "1", "aggregates": ["average", "sum"]}], + ([{"id": 1, "aggregates": ["average"]}, {"externalId": "1", "aggregates": ["average", "sum"]}], False), + ), + ], + ) + def test_process_time_series_input_ok(self, ids, external_ids, expected_output): + assert expected_output == DPS_CLIENT._process_ts_identifiers(ids, external_ids) + + @pytest.mark.parametrize( + "ids, external_ids, exception, match", + [ + (1.0, None, TypeError, "Invalid type ''"), + ([1.0], None, TypeError, "Invalid type ''"), + (None, 1, TypeError, "Invalid type ''"), + (None, [1], TypeError, "Invalid type ''"), + ({"wrong": 1, "aggregates": ["average"]}, None, ValueError, "Unknown key 'wrong'"), + (None, [{"externalId": 1, "wrong": ["average"]}], ValueError, "Unknown key 'wrong'"), + (None, {"id": 1, "aggregates": ["average"]}, ValueError, "Unknown key 'id'"), + ({"externalId": 1}, None, ValueError, "Unknown key 'externalId'"), + ], + ) + def test_process_time_series_input_fail(self, ids, external_ids, exception, match): + with pytest.raises(exception, match=match): + DPS_CLIENT._process_ts_identifiers(ids, external_ids) diff --git a/tests/tests_unit/test_api/test_events.py b/tests/tests_unit/test_api/test_events.py new file mode 100644 index 0000000000..0c3e166aa4 --- /dev/null +++ b/tests/tests_unit/test_api/test_events.py @@ -0,0 +1,168 @@ +import re + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.events import Event, EventList, EventUpdate +from cognite.client.data_classes import EventFilter +from tests.utils import jsgz_load + +EVENTS_API = CogniteClient().events + + +@pytest.fixture +def mock_events_response(rsps): + response_body = { + "items": [ + { + "externalId": "string", + "startTime": 0, + "endTime": 0, + "description": "string", + "metadata": {"metadata-key": "metadata-value"}, + "assetIds": [1], + "source": "string", + "id": 1, + "lastUpdatedTime": 0, + } + ] + } + + url_pattern = re.compile(re.escape(EVENTS_API._base_url) + "/.+") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestEvents: + def test_retrieve_single(self, mock_events_response): + res = EVENTS_API.retrieve(id=1) + assert isinstance(res, Event) + assert mock_events_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_retrieve_multiple(self, mock_events_response): + res = EVENTS_API.retrieve_multiple(ids=[1]) + assert isinstance(res, EventList) + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list(self, mock_events_response): + res = EVENTS_API.list(source="bla") + assert "bla" == jsgz_load(mock_events_response.calls[0].request.body)["filter"]["source"] + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create_single(self, mock_events_response): + res = EVENTS_API.create(Event(external_id="1")) + assert isinstance(res, Event) + assert mock_events_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_events_response): + res = EVENTS_API.create([Event(external_id="1")]) + assert isinstance(res, EventList) + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_iter_single(self, mock_events_response): + for event in EVENTS_API: + assert mock_events_response.calls[0].response.json()["items"][0] == event.dump(camel_case=True) + + def test_iter_chunk(self, mock_events_response): + for events in EVENTS_API(chunk_size=1): + assert mock_events_response.calls[0].response.json()["items"] == events.dump(camel_case=True) + + def test_delete_single(self, mock_events_response): + res = EVENTS_API.delete(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_events_response.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_events_response): + res = EVENTS_API.delete(id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_events_response.calls[0].request.body) + assert res is None + + def test_update_with_resource_class(self, mock_events_response): + res = EVENTS_API.update(Event(id=1)) + assert isinstance(res, Event) + assert mock_events_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_with_update_class(self, mock_events_response): + res = EVENTS_API.update(EventUpdate(id=1).description.set("blabla")) + assert isinstance(res, Event) + assert mock_events_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_multiple(self, mock_events_response): + res = EVENTS_API.update([EventUpdate(id=1).description.set("blabla")]) + assert isinstance(res, EventList) + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_search(self, mock_events_response): + res = EVENTS_API.search(filter=EventFilter(external_id_prefix="abc")) + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"search": {"description": None}, "filter": {"externalIdPrefix": "abc"}, "limit": None} == jsgz_load( + mock_events_response.calls[0].request.body + ) + + @pytest.mark.parametrize("filter_field", ["external_id_prefix", "externalIdPrefix"]) + def test_search_dict_filter(self, mock_events_response, filter_field): + res = EVENTS_API.search(filter={filter_field: "bla"}) + assert mock_events_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"search": {"description": None}, "filter": {"externalIdPrefix": "bla"}, "limit": None} == jsgz_load( + mock_events_response.calls[0].request.body + ) + + def test_event_update_object(self): + + assert isinstance( + EventUpdate(1) + .asset_ids.add([]) + .asset_ids.remove([]) + .asset_ids.set([]) + .description.set("") + .description.set(None) + .end_time.set(1) + .end_time.set(None) + .external_id.set("1") + .external_id.set(None) + .metadata.add({}) + .metadata.set({}) + .metadata.remove([]) + .source.set(1) + .source.set(None) + .start_time.set(1) + .start_time.set(None), + EventUpdate, + ) + + +@pytest.fixture +def mock_events_empty(rsps): + url_pattern = re.compile(re.escape(EVENTS_API._base_url) + "/.+") + rsps.add(rsps.POST, url_pattern, status=200, json={"items": []}) + yield rsps + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_event_list_to_pandas(self, mock_events_response): + import pandas as pd + + df = EVENTS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert 1 == df.shape[0] + assert {"metadata-key": "metadata-value"} == df["metadata"][0] + + def test_event_list_to_pandas_empty(self, mock_events_empty): + import pandas as pd + + df = EVENTS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert df.empty + + def test_event_to_pandas(self, mock_events_response): + import pandas as pd + + df = EVENTS_API.retrieve(id=1).to_pandas() + assert isinstance(df, pd.DataFrame) + assert "metadata" not in df.columns + assert [1] == df.loc["assetIds"][0] + assert "metadata-value" == df.loc["metadata-key"][0] diff --git a/tests/tests_unit/test_api/test_files.py b/tests/tests_unit/test_api/test_files.py new file mode 100644 index 0000000000..2dd6b53d64 --- /dev/null +++ b/tests/tests_unit/test_api/test_files.py @@ -0,0 +1,361 @@ +import json +import os +import re +from tempfile import TemporaryDirectory + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.files import FileMetadata, FileMetadataList, FileMetadataUpdate +from cognite.client.data_classes import FileMetadataFilter +from cognite.client.exceptions import CogniteAPIError +from tests.utils import jsgz_load, set_request_limit + +FILES_API = CogniteClient(max_workers=1).files + + +@pytest.fixture +def mock_files_response(rsps): + response_body = { + "items": [ + { + "externalId": "string", + "name": "string", + "source": "string", + "mimeType": "string", + "metadata": {"metadata-key": "metadata-value"}, + "assetIds": [1], + "id": 1, + "uploaded": True, + "uploadedTime": 0, + "createdTime": 0, + "lastUpdatedTime": 0, + } + ] + } + + url_pattern = re.compile(re.escape(FILES_API._base_url) + "/.+") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_file_upload_response(rsps): + response_body = { + "externalId": "string", + "name": "string", + "source": "string", + "mimeType": "string", + "metadata": {}, + "assetIds": [1], + "id": 1, + "uploaded": True, + "uploadedTime": 0, + "createdTime": 0, + "lastUpdatedTime": 0, + "uploadUrl": "https://upload.here", + } + rsps.add(rsps.POST, FILES_API._base_url + "/files", status=200, json=response_body) + rsps.add(rsps.PUT, "https://upload.here", status=200) + yield rsps + + +@pytest.fixture +def mock_file_download_response(rsps): + rsps.add( + rsps.POST, + FILES_API._base_url + "/files/byids", + status=200, + json={"items": [{"id": 1, "name": "file1"}, {"externalId": "2", "name": "file2"}]}, + ) + + def download_link_callback(request): + identifier = jsgz_load(request.body)["items"][0] + response = {} + if "id" in identifier: + response = {"items": [{"id": 1, "downloadUrl": "https://download.file1.here"}]} + elif "externalId" in identifier: + response = {"items": [{"externalId": "2", "downloadUrl": "https://download.file2.here"}]} + return 200, {}, json.dumps(response) + + rsps.add_callback( + rsps.POST, + FILES_API._base_url + "/files/downloadlink", + callback=download_link_callback, + content_type="application/json", + ) + rsps.add(rsps.GET, "https://download.file1.here", status=200, body="content1") + rsps.add(rsps.GET, "https://download.file2.here", status=200, body="content2") + yield rsps + + +@pytest.fixture +def mock_file_download_response_one_fails(rsps): + rsps.add( + rsps.POST, + FILES_API._base_url + "/files/byids", + status=200, + json={ + "items": [ + {"id": 1, "externalId": "success", "name": "file1"}, + {"externalId": "fail", "id": 2, "name": "file2"}, + ] + }, + ) + + def download_link_callback(request): + identifier = jsgz_load(request.body)["items"][0] + response = {} + if "id" in identifier: + return 200, {}, json.dumps({"items": [{"id": 1, "downloadUrl": "https://download.file1.here"}]}) + elif "externalId" in identifier: + return (400, {}, json.dumps({"error": {"message": "User error", "code": 400}})) + + rsps.add_callback( + rsps.POST, + FILES_API._base_url + "/files/downloadlink", + callback=download_link_callback, + content_type="application/json", + ) + rsps.add(rsps.GET, "https://download.file1.here", status=200, body="content1") + yield rsps + + +class TestFilesAPI: + def test_retrieve_single(self, mock_files_response): + res = FILES_API.retrieve(id=1) + assert isinstance(res, FileMetadata) + assert mock_files_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_retrieve_multiple(self, mock_files_response): + res = FILES_API.retrieve_multiple(ids=[1]) + assert isinstance(res, FileMetadataList) + assert mock_files_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list(self, mock_files_response): + res = FILES_API.list(source="bla", limit=10) + assert isinstance(res, FileMetadataList) + assert mock_files_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert "bla" == jsgz_load(mock_files_response.calls[0].request.body)["filter"]["source"] + assert 10 == jsgz_load(mock_files_response.calls[0].request.body)["limit"] + + def test_delete_single(self, mock_files_response): + res = FILES_API.delete(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_files_response.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_files_response): + res = FILES_API.delete(id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_files_response.calls[0].request.body) + assert res is None + + def test_update_with_resource_class(self, mock_files_response): + res = FILES_API.update(FileMetadata(id=1, source="bla")) + assert isinstance(res, FileMetadata) + assert {"items": [{"id": 1, "update": {"source": {"set": "bla"}}}]} == jsgz_load( + mock_files_response.calls[0].request.body + ) + + def test_update_with_update_class(self, mock_files_response): + res = FILES_API.update(FileMetadataUpdate(id=1).source.set("bla")) + assert isinstance(res, FileMetadata) + assert {"items": [{"id": 1, "update": {"source": {"set": "bla"}}}]} == jsgz_load( + mock_files_response.calls[0].request.body + ) + + def test_update_multiple(self, mock_files_response): + res = FILES_API.update([FileMetadataUpdate(id=1).source.set(None), FileMetadata(external_id="2", source="bla")]) + assert isinstance(res, FileMetadataList) + assert { + "items": [ + {"id": 1, "update": {"source": {"setNull": True}}}, + {"externalId": "2", "update": {"source": {"set": "bla"}}}, + ] + } == jsgz_load(mock_files_response.calls[0].request.body) + + def test_iter_single(self, mock_files_response): + for file in FILES_API: + assert isinstance(file, FileMetadata) + assert mock_files_response.calls[0].response.json()["items"][0] == file.dump(camel_case=True) + + def test_iter_chunk(self, mock_files_response): + for file in FILES_API(chunk_size=1): + assert isinstance(file, FileMetadataList) + assert mock_files_response.calls[0].response.json()["items"] == file.dump(camel_case=True) + + def test_search(self, mock_files_response): + res = FILES_API.search(filter=FileMetadataFilter(external_id_prefix="abc")) + assert mock_files_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"search": {"name": None}, "filter": {"externalIdPrefix": "abc"}, "limit": None} == jsgz_load( + mock_files_response.calls[0].request.body + ) + + @pytest.mark.parametrize("filter_field", ["external_id_prefix", "externalIdPrefix"]) + def test_search_dict_filter(self, mock_files_response, filter_field): + res = FILES_API.search(filter={filter_field: "abc"}) + assert mock_files_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert {"search": {"name": None}, "filter": {"externalIdPrefix": "abc"}, "limit": None} == jsgz_load( + mock_files_response.calls[0].request.body + ) + + def test_upload(self, mock_file_upload_response): + path = os.path.join(os.path.dirname(__file__), "files_for_test_upload", "file_for_test_upload_1.txt") + res = FILES_API.upload(path, name="bla") + response_body = mock_file_upload_response.calls[0].response.json() + del response_body["uploadUrl"] + assert FileMetadata._load(response_body) == res + assert "https://upload.here/" == mock_file_upload_response.calls[1].request.url + assert {"name": "bla"} == jsgz_load(mock_file_upload_response.calls[0].request.body) + assert b"content1\n" == mock_file_upload_response.calls[1].request.body + + def test_upload_no_name(self, mock_file_upload_response): + path = os.path.join(os.path.dirname(__file__), "files_for_test_upload", "file_for_test_upload_1.txt") + FILES_API.upload(path) + assert {"name": "file_for_test_upload_1.txt"} == jsgz_load(mock_file_upload_response.calls[0].request.body) + + def test_upload_from_directory(self, mock_file_upload_response): + path = os.path.join(os.path.dirname(__file__), "files_for_test_upload") + res = FILES_API.upload(path=path) + response_body = mock_file_upload_response.calls[0].response.json() + del response_body["uploadUrl"] + assert FileMetadataList([FileMetadata._load(response_body), FileMetadata._load(response_body)]) == res + assert 4 == len(mock_file_upload_response.calls) + for call in mock_file_upload_response.calls: + payload = call.request.body + if payload in [b"content1\n", b"content2\n"]: + continue + elif jsgz_load(payload)["name"] in ["file_for_test_upload_1.txt", "file_for_test_upload_2.txt"]: + continue + else: + raise AssertionError("incorrect payload: {}".format(payload)) + + def test_upload_from_directory_fails(self, rsps): + rsps.add(rsps.POST, FILES_API._base_url + "/files", status=400, json={}) + path = os.path.join(os.path.dirname(__file__), "files_for_test_upload") + with pytest.raises(CogniteAPIError) as e: + FILES_API.upload(path=path) + assert "file_for_test_upload_1.txt" in e.value.failed + assert "file_for_test_upload_2.txt" in e.value.failed + + def test_upload_from_directory_recursively(self, mock_file_upload_response): + path = os.path.join(os.path.dirname(__file__), "files_for_test_upload") + res = FILES_API.upload(path=path, recursive=True) + response_body = mock_file_upload_response.calls[0].response.json() + del response_body["uploadUrl"] + assert FileMetadataList([FileMetadata._load(response_body) for _ in range(3)]) == res + assert 6 == len(mock_file_upload_response.calls) + for call in mock_file_upload_response.calls: + payload = call.request.body + if payload in [b"content1\n", b"content2\n", b"content3\n"]: + continue + elif jsgz_load(payload)["name"] in [ + "file_for_test_upload_1.txt", + "file_for_test_upload_2.txt", + "file_for_test_upload_3.txt", + ]: + continue + else: + raise AssertionError("incorrect payload: {}".format(payload)) + + def test_upload_from_memory(self, mock_file_upload_response): + res = FILES_API.upload_bytes(content=b"content", name="bla") + response_body = mock_file_upload_response.calls[0].response.json() + del response_body["uploadUrl"] + assert FileMetadata._load(response_body) == res + assert "https://upload.here/" == mock_file_upload_response.calls[1].request.url + assert {"name": "bla"} == jsgz_load(mock_file_upload_response.calls[0].request.body) + assert b"content" == mock_file_upload_response.calls[1].request.body + + def test_upload_path_does_not_exist(self): + with pytest.raises(ValueError, match="does not exist"): + FILES_API.upload(path="/no/such/path") + + def test_download(self, mock_file_download_response): + with TemporaryDirectory() as dir: + res = FILES_API.download(directory=dir, id=[1], external_id=["2"]) + assert {"items": [{"id": 1}, {"externalId": "2"}]} == jsgz_load( + mock_file_download_response.calls[0].request.body + ) + assert res is None + assert os.path.isfile(os.path.join(dir, "file1")) + assert os.path.isfile(os.path.join(dir, "file2")) + + def test_download_one_file_fails(self, mock_file_download_response_one_fails): + with TemporaryDirectory() as dir: + with pytest.raises(CogniteAPIError) as e: + FILES_API.download(directory=dir, id=[1], external_id="fail") + assert [FileMetadata(id=1, name="file1", external_id="success")] == e.value.successful + assert [FileMetadata(id=2, name="file2", external_id="fail")] == e.value.failed + assert os.path.isfile(os.path.join(dir, "file1")) + + def test_download_to_memory(self, mock_file_download_response): + mock_file_download_response.assert_all_requests_are_fired = False + res = FILES_API.download_bytes(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_file_download_response.calls[0].request.body) + assert res == b"content1" + + def test_download_ids_over_limit(self, mock_file_download_response): + with set_request_limit(FILES_API, 1): + with TemporaryDirectory() as dir: + res = FILES_API.download(directory=dir, id=[1], external_id=["2"]) + bodies = [jsgz_load(mock_file_download_response.calls[i].request.body) for i in range(2)] + assert {"items": [{"id": 1}]} in bodies + assert {"items": [{"externalId": "2"}]} in bodies + assert res is None + assert os.path.isfile(os.path.join(dir, "file1")) + assert os.path.isfile(os.path.join(dir, "file2")) + + def test_files_update_object(self): + assert isinstance( + FileMetadataUpdate(1) + .asset_ids.add([]) + .asset_ids.remove([]) + .asset_ids.set([]) + .asset_ids.set(None) + .external_id.set("1") + .external_id.set(None) + .metadata.add({}) + .metadata.remove([]) + .metadata.set({}) + .metadata.set(None) + .source.set(1) + .source.set(None), + FileMetadataUpdate, + ) + + +@pytest.fixture +def mock_files_empty(rsps): + url_pattern = re.compile(re.escape(FILES_API._base_url) + "/.+") + rsps.add(rsps.POST, url_pattern, status=200, json={"items": []}) + yield rsps + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_file_list_to_pandas(self, mock_files_response): + import pandas as pd + + df = FILES_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert 1 == df.shape[0] + assert {"metadata-key": "metadata-value"} == df["metadata"][0] + + def test_file_list_to_pandas_empty(self, mock_files_empty): + import pandas as pd + + df = FILES_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert df.empty + + def test_file_to_pandas(self, mock_files_response): + import pandas as pd + + df = FILES_API.retrieve(id=1).to_pandas() + assert isinstance(df, pd.DataFrame) + assert "metadata" not in df.columns + assert [1] == df.loc["assetIds"][0] + assert "metadata-value" == df.loc["metadata-key"][0] diff --git a/tests/tests_unit/test_api/test_iam.py b/tests/tests_unit/test_api/test_iam.py new file mode 100644 index 0000000000..d24b13ab47 --- /dev/null +++ b/tests/tests_unit/test_api/test_iam.py @@ -0,0 +1,228 @@ +import re + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.iam import APIKeyList, GroupList, SecurityCategoryList, ServiceAccountList +from cognite.client.data_classes import APIKey, Group, SecurityCategory, ServiceAccount +from tests.utils import jsgz_load + +IAM_API = CogniteClient().iam + + +@pytest.fixture +def mock_service_accounts(rsps): + response_body = { + "items": [{"name": "service@bla.com", "groups": [1, 2, 3], "id": 0, "isDeleted": False, "deletedTime": 0}] + } + url_pattern = re.compile(re.escape(IAM_API._base_url) + "/serviceaccounts.*") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestServiceAccounts: + def test_list(self, mock_service_accounts): + res = IAM_API.service_accounts.list() + assert isinstance(res, ServiceAccountList) + assert mock_service_accounts.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create(self, mock_service_accounts): + res = IAM_API.service_accounts.create(ServiceAccount(name="service@bla.com", groups=[1, 2, 3])) + assert isinstance(res, ServiceAccount) + assert {"items": [{"name": "service@bla.com", "groups": [1, 2, 3]}]} == jsgz_load( + mock_service_accounts.calls[0].request.body + ) + assert mock_service_accounts.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_service_accounts): + res = IAM_API.service_accounts.create([ServiceAccount(name="service@bla.com", groups=[1, 2, 3])]) + assert isinstance(res, ServiceAccountList) + assert {"items": [{"name": "service@bla.com", "groups": [1, 2, 3]}]} == jsgz_load( + mock_service_accounts.calls[0].request.body + ) + assert mock_service_accounts.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_delete(self, mock_service_accounts): + res = IAM_API.service_accounts.delete(1) + assert {"items": [1]} == jsgz_load(mock_service_accounts.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_service_accounts): + res = IAM_API.service_accounts.delete([1]) + assert {"items": [1]} == jsgz_load(mock_service_accounts.calls[0].request.body) + assert res is None + + +@pytest.fixture +def mock_api_keys(rsps): + response_body = {"items": [{"id": 1, "serviceAccountId": 1, "createdTime": 0, "status": "ACTIVE"}]} + url_pattern = re.compile(re.escape(IAM_API._base_url) + "/apikeys.*") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestAPIKeys: + def test_list(self, mock_api_keys): + res = IAM_API.api_keys.list() + assert isinstance(res, APIKeyList) + assert mock_api_keys.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create(self, mock_api_keys): + res = IAM_API.api_keys.create(1) + assert isinstance(res, APIKey) + assert {"items": [{"serviceAccountId": 1}]} == jsgz_load(mock_api_keys.calls[0].request.body) + assert mock_api_keys.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_api_keys): + res = IAM_API.api_keys.create([1]) + assert isinstance(res, APIKeyList) + assert {"items": [{"serviceAccountId": 1}]} == jsgz_load(mock_api_keys.calls[0].request.body) + assert mock_api_keys.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_delete(self, mock_api_keys): + res = IAM_API.api_keys.delete(1) + assert {"items": [1]} == jsgz_load(mock_api_keys.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_api_keys): + res = IAM_API.api_keys.delete([1]) + assert {"items": [1]} == jsgz_load(mock_api_keys.calls[0].request.body) + assert res is None + + +@pytest.fixture +def mock_groups(rsps): + response_body = { + "items": [ + { + "name": "Production Engineers", + "sourceId": "b7c9a5a4-99c2-4785-bed3-5e6ad9a78603", + "capabilities": [{"groupsAcl": {"actions": ["LIST"], "scope": {"all": {}}}}], + "id": 0, + "isDeleted": False, + "deletedTime": 0, + } + ] + } + url_pattern = re.compile(re.escape(IAM_API._base_url) + "/groups.*") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_group_service_account_response(rsps): + response_body = { + "items": [ + { + "name": "some-internal-service@apple.com", + "groups": [1, 2, 3], + "id": 0, + "isDeleted": False, + "deletedTime": 0, + } + ] + } + url_pattern = re.compile(re.escape(IAM_API._base_url) + "/groups/1/serviceaccounts.*") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestGroups: + def test_list(self, mock_groups): + res = IAM_API.groups.list() + assert isinstance(res, GroupList) + assert mock_groups.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create(self, mock_groups): + res = IAM_API.groups.create(1) + assert isinstance(res, Group) + assert {"items": [1]} == jsgz_load(mock_groups.calls[0].request.body) + assert mock_groups.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_groups): + res = IAM_API.groups.create([1]) + assert isinstance(res, GroupList) + assert {"items": [1]} == jsgz_load(mock_groups.calls[0].request.body) + assert mock_groups.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_delete(self, mock_groups): + res = IAM_API.groups.delete(1) + assert {"items": [1]} == jsgz_load(mock_groups.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_groups): + res = IAM_API.groups.delete([1]) + assert {"items": [1]} == jsgz_load(mock_groups.calls[0].request.body) + assert res is None + + def test_list_service_accounts(self, mock_group_service_account_response): + res = IAM_API.groups.list_service_accounts(1) + assert isinstance(res, ServiceAccountList) + assert mock_group_service_account_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_add_service_account(self, mock_group_service_account_response): + res = IAM_API.groups.add_service_account(1, 1) + assert res is None + assert {"items": [1]} == jsgz_load(mock_group_service_account_response.calls[0].request.body) + + def test_add_service_account_multiple(self, mock_group_service_account_response): + res = IAM_API.groups.add_service_account(1, [1]) + assert res is None + assert {"items": [1]} == jsgz_load(mock_group_service_account_response.calls[0].request.body) + + def test_remove_service_account(self, mock_group_service_account_response): + res = IAM_API.groups.remove_service_account(1, 1) + assert res is None + assert {"items": [1]} == jsgz_load(mock_group_service_account_response.calls[0].request.body) + + def test_remove_service_account_multiple(self, mock_group_service_account_response): + res = IAM_API.groups.remove_service_account(1, [1]) + assert res is None + assert {"items": [1]} == jsgz_load(mock_group_service_account_response.calls[0].request.body) + + +@pytest.fixture +def mock_security_categories(rsps): + response_body = {"items": [{"name": "bla", "id": 1}]} + url_pattern = re.compile(re.escape(IAM_API._base_url) + "/securitycategories.*") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestSecurityCategories: + def test_list(self, mock_security_categories): + res = IAM_API.security_categories.list() + assert isinstance(res, SecurityCategoryList) + assert mock_security_categories.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_create(self, mock_security_categories): + res = IAM_API.security_categories.create(1) + assert isinstance(res, SecurityCategory) + assert {"items": [1]} == jsgz_load(mock_security_categories.calls[0].request.body) + assert mock_security_categories.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_security_categories): + res = IAM_API.security_categories.create([1]) + assert isinstance(res, SecurityCategoryList) + assert {"items": [1]} == jsgz_load(mock_security_categories.calls[0].request.body) + assert mock_security_categories.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_delete(self, mock_security_categories): + res = IAM_API.security_categories.delete(1) + assert {"items": [1]} == jsgz_load(mock_security_categories.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_security_categories): + res = IAM_API.security_categories.delete([1]) + assert {"items": [1]} == jsgz_load(mock_security_categories.calls[0].request.body) + assert res is None diff --git a/tests/tests_unit/test_api/test_raw.py b/tests/tests_unit/test_api/test_raw.py new file mode 100644 index 0000000000..2edd103817 --- /dev/null +++ b/tests/tests_unit/test_api/test_raw.py @@ -0,0 +1,273 @@ +import re + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.raw import Database, DatabaseList, Row, RowList, Table, TableList +from cognite.client.exceptions import CogniteAPIError +from tests.utils import jsgz_load + +COGNITE_CLIENT = CogniteClient() +RAW_API = COGNITE_CLIENT.raw + + +@pytest.fixture +def mock_raw_db_response(rsps): + response_body = {"items": [{"name": "db1"}]} + + url_pattern = re.compile(re.escape(RAW_API._base_url) + "/raw/dbs(?:/delete|$|\?.+)") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_raw_table_response(rsps): + response_body = {"items": [{"name": "table1"}]} + + url_pattern = re.compile(re.escape(RAW_API._base_url) + "/raw/dbs/db1/tables(?:/delete|$|\?.+)") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_raw_row_response(rsps): + response_body = {"items": [{"key": "row1", "columns": {"c1": 1, "c2": "2"}}]} + + url_pattern = re.compile(re.escape(RAW_API._base_url) + "/raw/dbs/db1/tables/table1/rows(?:/delete|/row1|$|\?.+)") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +@pytest.fixture +def mock_retrieve_raw_row_response(rsps): + response_body = {"key": "row1", "columns": {"c1": 1, "c2": "2"}} + rsps.add(rsps.GET, RAW_API._base_url + "/raw/dbs/db1/tables/table1/rows/row1", status=200, json=response_body) + yield rsps + + +class TestRawDatabases: + def test_create_single(self, mock_raw_db_response): + res = RAW_API.databases.create(name="db1") + assert isinstance(res, Database) + assert COGNITE_CLIENT == res._cognite_client + assert mock_raw_db_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + assert [{"name": "db1"}] == jsgz_load(mock_raw_db_response.calls[0].request.body)["items"] + + def test_create_multiple(self, mock_raw_db_response): + res_list = RAW_API.databases.create(name=["db1"]) + assert isinstance(res_list, DatabaseList) + for res in res_list: + assert COGNITE_CLIENT == res._cognite_client + assert COGNITE_CLIENT == res_list._cognite_client + assert [{"name": "db1"}] == jsgz_load(mock_raw_db_response.calls[0].request.body)["items"] + assert mock_raw_db_response.calls[0].response.json()["items"] == res_list.dump(camel_case=True) + + def test_list(self, mock_raw_db_response): + res_list = RAW_API.databases.list() + assert DatabaseList([Database("db1")]) == res_list + + def test_iter_single(self, mock_raw_db_response): + for db in RAW_API.databases: + assert mock_raw_db_response.calls[0].response.json()["items"][0] == db.dump(camel_case=True) + + def test_iter_chunk(self, mock_raw_db_response): + for db in RAW_API.databases(chunk_size=1): + assert mock_raw_db_response.calls[0].response.json()["items"] == db.dump(camel_case=True) + + def test_delete(self, mock_raw_db_response): + res = RAW_API.databases.delete(name="db1") + assert res is None + assert [{"name": "db1"}] == jsgz_load(mock_raw_db_response.calls[0].request.body)["items"] + + def test_delete_multiple(self, mock_raw_db_response): + res = RAW_API.databases.delete(name=["db1"]) + assert res is None + assert [{"name": "db1"}] == jsgz_load(mock_raw_db_response.calls[0].request.body)["items"] + + def test_delete_fail(self, rsps): + rsps.add( + rsps.POST, + RAW_API._base_url + "/raw/dbs/delete", + status=400, + json={"error": {"message": "User Error", "code": 400}}, + ) + with pytest.raises(CogniteAPIError) as e: + RAW_API.databases.delete("db1") + assert e.value.failed == ["db1"] + + def test_get_tables_in_db(self, mock_raw_db_response, mock_raw_table_response): + db = RAW_API.databases.list()[0] + tables = db.tables() + assert TableList([Table(name="table1")]) == tables + + +class TestRawTables: + def test_create_single(self, mock_raw_table_response): + res = RAW_API.tables.create("db1", name="table1") + assert isinstance(res, Table) + assert COGNITE_CLIENT == res._cognite_client + assert mock_raw_table_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + assert [{"name": "table1"}] == jsgz_load(mock_raw_table_response.calls[0].request.body)["items"] + assert "db1" == res._db_name + + def test_create_multiple(self, mock_raw_table_response): + res_list = RAW_API.tables.create("db1", name=["table1"]) + assert isinstance(res_list, TableList) + for res in res_list: + assert COGNITE_CLIENT == res._cognite_client + assert "db1" == res._db_name + assert COGNITE_CLIENT == res_list._cognite_client + assert [{"name": "table1"}] == jsgz_load(mock_raw_table_response.calls[0].request.body)["items"] + assert mock_raw_table_response.calls[0].response.json()["items"] == res_list.dump(camel_case=True) + + def test_list(self, mock_raw_table_response): + res_list = RAW_API.tables.list(db_name="db1") + for res in res_list: + assert "db1" == res._db_name + assert COGNITE_CLIENT == res._cognite_client + assert TableList([Table("table1")]) == res_list + + def test_iter_single(self, mock_raw_table_response): + for table in RAW_API.tables(db_name="db1"): + assert mock_raw_table_response.calls[0].response.json()["items"][0] == table.dump(camel_case=True) + + def test_iter_chunk(self, mock_raw_table_response): + for table_list in RAW_API.tables("db1", chunk_size=1): + for table in table_list: + assert "db1" == table._db_name + assert COGNITE_CLIENT == table._cognite_client + assert mock_raw_table_response.calls[0].response.json()["items"] == table_list.dump(camel_case=True) + + def test_delete(self, mock_raw_table_response): + res = RAW_API.tables.delete("db1", name="table1") + assert res is None + assert [{"name": "table1"}] == jsgz_load(mock_raw_table_response.calls[0].request.body)["items"] + + def test_delete_multiple(self, mock_raw_table_response): + res = RAW_API.tables.delete(db_name="db1", name=["table1"]) + assert res is None + assert [{"name": "table1"}] == jsgz_load(mock_raw_table_response.calls[0].request.body)["items"] + + def test_delete_fail(self, rsps): + rsps.add( + rsps.POST, + RAW_API._base_url + "/raw/dbs/db1/tables/delete", + status=400, + json={"error": {"message": "User Error", "code": 400}}, + ) + with pytest.raises(CogniteAPIError) as e: + RAW_API.tables.delete("db1", "table1") + assert e.value.failed == ["table1"] + + def test_get_rows_in_table(self, mock_raw_table_response, mock_raw_row_response): + tables = RAW_API.tables.list(db_name="db1") + rows = tables[0].rows() + assert RowList([Row._load({"key": "row1", "columns": {"c1": 1, "c2": "2"}})]) == rows + + +class TestRawRows: + def test_retrieve(self, mock_retrieve_raw_row_response): + res = RAW_API.rows.retrieve(db_name="db1", table_name="table1", key="row1") + assert mock_retrieve_raw_row_response.calls[0].response.json() == res.dump(camel_case=True) + assert mock_retrieve_raw_row_response.calls[0].request.url.endswith("/rows/row1") + + def test_insert_w_rows_as_dict(self, mock_raw_row_response): + res = RAW_API.rows.insert( + db_name="db1", table_name="table1", row={"row1": {"c1": 1, "c2": "2"}}, ensure_parent=True + ) + assert res is None + assert [{"key": "row1", "columns": {"c1": 1, "c2": "2"}}] == jsgz_load( + mock_raw_row_response.calls[0].request.body + )["items"] + + def test_insert_single_DTO(self, mock_raw_row_response): + res = RAW_API.rows.insert( + db_name="db1", table_name="table1", row=Row(key="row1", columns={"c1": 1, "c2": "2"}), ensure_parent=False + ) + assert res is None + assert [{"key": "row1", "columns": {"c1": 1, "c2": "2"}}] == jsgz_load( + mock_raw_row_response.calls[0].request.body + )["items"] + + def test_insert_multiple_DTO(self, mock_raw_row_response): + res = RAW_API.rows.insert("db1", "table1", row=[Row(key="row1", columns={"c1": 1, "c2": "2"})]) + assert res is None + assert [{"key": "row1", "columns": {"c1": 1, "c2": "2"}}] == jsgz_load( + mock_raw_row_response.calls[0].request.body + )["items"] + + def test_insert_fail(self, rsps): + rsps.add(rsps.POST, RAW_API._base_url + "/raw/dbs/db1/tables/table1/rows", status=400, json={}) + with pytest.raises(CogniteAPIError) as e: + RAW_API.rows.insert("db1", "table1", {"row1": {"c1": 1}}) + assert e.value.failed == ["row1"] + + def test_list(self, mock_raw_row_response): + res_list = RAW_API.rows.list(db_name="db1", table_name="table1") + assert RowList([Row(key="row1", columns={"c1": 1, "c2": "2"})]) == res_list + + def test_iter_single(self, mock_raw_row_response): + for db in RAW_API.rows(db_name="db1", table_name="table1"): + assert mock_raw_row_response.calls[0].response.json()["items"][0] == db.dump(camel_case=True) + + def test_iter_chunk(self, mock_raw_row_response): + for db in RAW_API.rows("db1", "table1", chunk_size=1): + assert mock_raw_row_response.calls[0].response.json()["items"] == db.dump(camel_case=True) + + def test_delete(self, mock_raw_row_response): + res = RAW_API.rows.delete("db1", table_name="table1", key="row1") + assert res is None + assert [{"key": "row1"}] == jsgz_load(mock_raw_row_response.calls[0].request.body)["items"] + + def test_delete_multiple(self, mock_raw_row_response): + res = RAW_API.rows.delete(db_name="db1", table_name="table1", key=["row1"]) + assert res is None + assert [{"key": "row1"}] == jsgz_load(mock_raw_row_response.calls[0].request.body)["items"] + + def test_delete_fail(self, rsps): + rsps.add( + rsps.POST, + RAW_API._base_url + "/raw/dbs/db1/tables/table1/rows/delete", + status=400, + json={"error": {"message": "User Error", "code": 400}}, + ) + with pytest.raises(CogniteAPIError) as e: + RAW_API.rows.delete("db1", "table1", "key1") + assert e.value.failed == ["key1"] + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_dbs_to_pandas(self): + import pandas as pd + + db_list = DatabaseList([Database("kar"), Database("car"), Database("dar")]) + + pd.testing.assert_frame_equal(pd.DataFrame({"name": ["kar", "car", "dar"]}), db_list.to_pandas()) + pd.testing.assert_frame_equal(pd.DataFrame({"value": ["kar"]}, index=["name"]), db_list[0].to_pandas()) + + def test_tables_to_pandas(self): + import pandas as pd + + table_list = TableList([Table("kar"), Table("car"), Table("dar")]) + + pd.testing.assert_frame_equal(pd.DataFrame({"name": ["kar", "car", "dar"]}), table_list.to_pandas()) + pd.testing.assert_frame_equal(pd.DataFrame({"value": ["kar"]}, index=["name"]), table_list[0].to_pandas()) + + def test_rows_to_pandas(self): + import pandas as pd + + row_list = RowList([Row("k1", {"c1": "v1", "c2": "v1"}), Row("k2", {"c1": "v2", "c2": "v2"})]) + pd.testing.assert_frame_equal( + pd.DataFrame({"c1": ["v1", "v2"], "c2": ["v1", "v2"]}, index=["k1", "k2"]), row_list.to_pandas() + ) + pd.testing.assert_frame_equal(pd.DataFrame({"c1": ["v1"], "c2": ["v1"]}, index=["k1"]), row_list[0].to_pandas()) diff --git a/tests/tests_unit/test_api/test_signatures.py b/tests/tests_unit/test_api/test_signatures.py new file mode 100644 index 0000000000..8d1ba31c18 --- /dev/null +++ b/tests/tests_unit/test_api/test_signatures.py @@ -0,0 +1,46 @@ +import inspect + +import pytest + +from cognite.client._api import assets, events, files + + +class TestListAndIterSignatures: + @pytest.mark.parametrize( + "api, filter", + [ + (assets.AssetsAPI, assets.AssetFilter), + (events.EventsAPI, events.EventFilter), + (files.FilesAPI, files.FileMetadataFilter), + ], + ) + def test_list_and_iter_signatures_same_as_filter_signature(self, api, filter): + iter_parameters = dict(inspect.signature(api.__call__).parameters) + del iter_parameters["chunk_size"] + list_parameters = dict(inspect.signature(api.list).parameters) + del list_parameters["limit"] + filter_parameters = dict(inspect.signature(filter.__init__).parameters) + del filter_parameters["cognite_client"] + assert iter_parameters == filter_parameters + assert list_parameters == filter_parameters + + +class TestFileMetadataUploadSignatures: + def test_upload_signatures_same_as_file_metadata_signature(self): + upload_parameters = dict(inspect.signature(files.FilesAPI.upload).parameters) + del upload_parameters["path"] + del upload_parameters["recursive"] + del upload_parameters["overwrite"] + upload_from_memory_parameters = dict(inspect.signature(files.FilesAPI.upload_bytes).parameters) + del upload_from_memory_parameters["content"] + del upload_from_memory_parameters["overwrite"] + file_metadata_parameters = dict(inspect.signature(files.FileMetadata.__init__).parameters) + del file_metadata_parameters["id"] + del file_metadata_parameters["uploaded_time"] + del file_metadata_parameters["created_time"] + del file_metadata_parameters["last_updated_time"] + del file_metadata_parameters["uploaded"] + del file_metadata_parameters["cognite_client"] + + assert upload_parameters == file_metadata_parameters + assert upload_from_memory_parameters == file_metadata_parameters diff --git a/tests/tests_unit/test_api/test_time_series.py b/tests/tests_unit/test_api/test_time_series.py new file mode 100644 index 0000000000..70b94bcf5d --- /dev/null +++ b/tests/tests_unit/test_api/test_time_series.py @@ -0,0 +1,252 @@ +import re +from unittest import mock + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes import TimeSeries, TimeSeriesFilter, TimeSeriesList, TimeSeriesUpdate +from tests.utils import jsgz_load + +COGNITE_CLIENT = CogniteClient() +TS_API = COGNITE_CLIENT.time_series + + +@pytest.fixture +def mock_ts_response(rsps): + response_body = { + "items": [ + { + "id": 0, + "externalId": "string", + "name": "stringname", + "isString": True, + "metadata": {"metadata-key": "metadata-value"}, + "unit": "string", + "assetId": 0, + "isStep": True, + "description": "string", + "securityCategories": [0], + "createdTime": 0, + "lastUpdatedTime": 0, + } + ] + } + url_pattern = re.compile(re.escape(TS_API._base_url) + "/timeseries(?:/byids|/update|/delete|/search|$|\?.+)") + rsps.assert_all_requests_are_fired = False + + rsps.add(rsps.POST, url_pattern, status=200, json=response_body) + rsps.add(rsps.GET, url_pattern, status=200, json=response_body) + yield rsps + + +class TestTimeSeries: + def test_retrieve_single(self, mock_ts_response): + res = TS_API.retrieve(id=1) + assert isinstance(res, TimeSeries) + assert mock_ts_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_retrieve_multiple(self, mock_ts_response): + res = TS_API.retrieve_multiple(ids=[1]) + assert isinstance(res, TimeSeriesList) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_list(self, mock_ts_response): + res = TS_API.list() + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + @pytest.mark.dsl + def test_list_with_asset_ids(self, mock_ts_response): + import numpy + + TS_API.list(asset_ids=[1]) + TS_API.list(asset_ids=[numpy.int64(1)]) + for i in range(len(mock_ts_response.calls)): + assert "assetIds=%5B1%5D" in mock_ts_response.calls[i].request.url + + def test_create_single(self, mock_ts_response): + res = TS_API.create(TimeSeries(external_id="1", name="blabla")) + assert isinstance(res, TimeSeries) + assert mock_ts_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_create_multiple(self, mock_ts_response): + res = TS_API.create([TimeSeries(external_id="1", name="blabla")]) + assert isinstance(res, TimeSeriesList) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_iter_single(self, mock_ts_response): + for asset in TS_API: + assert mock_ts_response.calls[0].response.json()["items"][0] == asset.dump(camel_case=True) + + def test_iter_chunk(self, mock_ts_response): + for assets in TS_API(chunk_size=1): + assert mock_ts_response.calls[0].response.json()["items"] == assets.dump(camel_case=True) + + def test_delete_single(self, mock_ts_response): + res = TS_API.delete(id=1) + assert {"items": [{"id": 1}]} == jsgz_load(mock_ts_response.calls[0].request.body) + assert res is None + + def test_delete_multiple(self, mock_ts_response): + res = TS_API.delete(id=[1]) + assert {"items": [{"id": 1}]} == jsgz_load(mock_ts_response.calls[0].request.body) + assert res is None + + def test_update_with_resource_class(self, mock_ts_response): + res = TS_API.update(TimeSeries(id=1)) + assert isinstance(res, TimeSeries) + assert mock_ts_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_with_update_class(self, mock_ts_response): + res = TS_API.update(TimeSeriesUpdate(id=1).description.set("blabla")) + assert isinstance(res, TimeSeries) + assert mock_ts_response.calls[0].response.json()["items"][0] == res.dump(camel_case=True) + + def test_update_multiple(self, mock_ts_response): + res = TS_API.update([TimeSeriesUpdate(id=1).description.set("blabla")]) + assert isinstance(res, TimeSeriesList) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + + def test_search(self, mock_ts_response): + res = TS_API.search(filter=TimeSeriesFilter(is_string=True)) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert { + "search": {"name": None, "description": None, "query": None}, + "filter": {"isString": True}, + "limit": None, + } == jsgz_load(mock_ts_response.calls[0].request.body) + + @pytest.mark.parametrize("filter_field", ["is_string", "isString"]) + def test_search_dict_filter(self, mock_ts_response, filter_field): + res = TS_API.search(filter={filter_field: True}) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + assert { + "search": {"name": None, "description": None, "query": None}, + "filter": {"isString": True}, + "limit": None, + } == jsgz_load(mock_ts_response.calls[0].request.body) + + def test_search_with_filter(self, mock_ts_response): + res = TS_API.search(name="n", description="d", query="q", filter=TimeSeriesFilter(unit="bla")) + assert mock_ts_response.calls[0].response.json()["items"] == res.dump(camel_case=True) + req_body = jsgz_load(mock_ts_response.calls[0].request.body) + assert "bla" == req_body["filter"]["unit"] + assert {"name": "n", "description": "d", "query": "q"} == req_body["search"] + + def test_events_update_object(self): + assert isinstance( + TimeSeriesUpdate(1) + .asset_id.set(1) + .asset_id.set(None) + .description.set("") + .description.set(None) + .external_id.set("1") + .external_id.set(None) + .metadata.set({}) + .metadata.add({}) + .metadata.remove([]) + .name.set("") + .name.set(None) + .security_categories.set([]) + .security_categories.add([]) + .security_categories.remove([]) + .unit.set("") + .unit.set(None), + TimeSeriesUpdate, + ) + + +@pytest.mark.dsl +class TestPlotTimeSeries: + @pytest.fixture + def mock_get_dps(self, rsps): + rsps.add( + rsps.POST, + TS_API._base_url + "/timeseries/data/list", + status=200, + json={ + "items": [ + { + "id": 0, + "externalId": "string1", + "datapoints": [{"timestamp": i * 10000, "average": i} for i in range(5000)], + } + ] + }, + ) + + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.rename") + def test_plot_time_series_name_labels(self, pandas_rename_mock, plt_show_mock, mock_ts_response, mock_get_dps): + res = TS_API.retrieve(id=0) + df_mock = mock.MagicMock() + pandas_rename_mock.return_value = df_mock + res.plot(aggregates=["average"], granularity="1d") + + assert {0: "stringname", "0|average": "stringname|average"} == pandas_rename_mock.call_args[1]["columns"] + assert 1 == df_mock.plot.call_count + assert 1 == plt_show_mock.call_count + + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.plot") + def test_plot_time_series_id_labels(self, pandas_plot_mock, plt_show_mock, mock_ts_response, mock_get_dps): + res = TS_API.retrieve(id=0) + res.plot(id_labels=True, aggregates=["average"], granularity="1s") + + assert 1 == pandas_plot_mock.call_count + assert 1 == plt_show_mock.call_count + + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.rename") + def test_plot_time_series_list_name_labels(self, pandas_rename_mock, plt_show_mock, mock_ts_response, mock_get_dps): + res = TS_API.retrieve_multiple(ids=[0]) + df_mock = mock.MagicMock() + pandas_rename_mock.return_value = df_mock + res.plot(aggregates=["average"], granularity="1h") + assert {0: "stringname", "0|average": "stringname|average"} == pandas_rename_mock.call_args[1]["columns"] + assert 1 == df_mock.plot.call_count + assert 1 == plt_show_mock.call_count + + @mock.patch("matplotlib.pyplot.show") + @mock.patch("pandas.core.frame.DataFrame.plot") + def test_plot_time_series_list_id_labels(self, pandas_plot_mock, plt_show_mock, mock_ts_response, mock_get_dps): + res = TS_API.retrieve_multiple(ids=[0]) + res.plot(id_labels=True) + + assert 1 == pandas_plot_mock.call_count + assert 1 == plt_show_mock.call_count + + +@pytest.fixture +def mock_time_series_empty(rsps): + url_pattern = re.compile(re.escape(TS_API._base_url) + "/.+") + rsps.assert_all_requests_are_fired = False + rsps.add(rsps.POST, url_pattern, status=200, json={"items": []}) + rsps.add(rsps.GET, url_pattern, status=200, json={"items": []}) + yield rsps + + +@pytest.mark.dsl +class TestPandasIntegration: + def test_time_series_list_to_pandas(self, mock_ts_response): + import pandas as pd + + df = TS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert 1 == df.shape[0] + assert {"metadata-key": "metadata-value"} == df["metadata"][0] + + def test_time_series_list_to_pandas_empty(self, mock_time_series_empty): + import pandas as pd + + df = TS_API.list().to_pandas() + assert isinstance(df, pd.DataFrame) + assert df.empty + + def test_time_series_to_pandas(self, mock_ts_response): + import pandas as pd + + df = TS_API.retrieve(id=1).to_pandas() + assert isinstance(df, pd.DataFrame) + assert "metadata" not in df.columns + assert [0] == df.loc["securityCategories"][0] + assert "metadata-value" == df.loc["metadata-key"][0] diff --git a/tests/tests_unit/test_api_client.py b/tests/tests_unit/test_api_client.py new file mode 100644 index 0000000000..b44104158b --- /dev/null +++ b/tests/tests_unit/test_api_client.py @@ -0,0 +1,796 @@ +import os +from collections import namedtuple + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api_client import APIClient, _get_status_codes_to_retry +from cognite.client.data_classes._base import * +from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError +from tests.utils import jsgz_load, set_request_limit + +BASE_URL = "http://localtest.com/api/1.0/projects/test-project" +URL_PATH = "/someurl" + +RESPONSE = {"any": "ok"} +COGNITE_CLIENT = CogniteClient() +API_CLIENT = APIClient( + project="test-project", + api_key="abc", + base_url=BASE_URL, + max_workers=1, + headers={"x-cdp-app": "python-sdk-integration-tests"}, + timeout=60, + cognite_client=COGNITE_CLIENT, +) + + +class TestBasicRequests: + @pytest.fixture + def mock_all_requests_ok(self, rsps): + rsps.assert_all_requests_are_fired = False + for method in [rsps.GET, rsps.PUT, rsps.POST, rsps.DELETE]: + rsps.add(method, BASE_URL + URL_PATH, status=200, json=RESPONSE) + yield rsps + + @pytest.fixture + def mock_all_requests_fail(self, rsps): + rsps.assert_all_requests_are_fired = False + for method in [rsps.GET, rsps.PUT, rsps.POST, rsps.DELETE]: + rsps.add(method, BASE_URL + URL_PATH, status=400, json={"error": "Client error"}) + rsps.add(method, BASE_URL + URL_PATH, status=500, body="Server error") + rsps.add(method, BASE_URL + URL_PATH, status=500, json={"error": "Server error"}) + rsps.add(method, BASE_URL + URL_PATH, status=400, json={"error": {"code": 400, "message": "Client error"}}) + + RequestCase = namedtuple("RequestCase", ["name", "method", "kwargs"]) + + request_cases = [ + RequestCase(name="post", method=API_CLIENT._post, kwargs={"url_path": URL_PATH, "json": {"any": "ok"}}), + RequestCase(name="get", method=API_CLIENT._get, kwargs={"url_path": URL_PATH}), + RequestCase(name="delete", method=API_CLIENT._delete, kwargs={"url_path": URL_PATH}), + RequestCase(name="put", method=API_CLIENT._put, kwargs={"url_path": URL_PATH, "json": {"any": "ok"}}), + ] + + @pytest.mark.parametrize("name, method, kwargs", request_cases) + def test_requests_ok(self, name, method, kwargs, mock_all_requests_ok): + response = method(**kwargs) + assert response.status_code == 200 + assert response.json() == RESPONSE + + request_headers = mock_all_requests_ok.calls[0].request.headers + assert "application/json" == request_headers["content-type"] + assert "application/json" == request_headers["accept"] + assert API_CLIENT._api_key == request_headers["api-key"] + assert "python-sdk-integration-tests" == request_headers["x-cdp-app"] + assert "User-Agent" in request_headers + + @pytest.mark.usefixtures("mock_all_requests_fail") + @pytest.mark.parametrize("name, method, kwargs", request_cases) + def test_requests_fail(self, name, method, kwargs): + with pytest.raises(CogniteAPIError, match="Client error") as e: + method(**kwargs) + assert e.value.code == 400 + + with pytest.raises(CogniteAPIError, match="Server error") as e: + method(**kwargs) + assert e.value.code == 500 + + with pytest.raises(CogniteAPIError, match="Server error") as e: + method(**kwargs) + assert e.value.code == 500 + + with pytest.raises(CogniteAPIError, match="Client error | code: 400 | X-Request-ID:") as e: + method(**kwargs) + assert e.value.code == 400 + assert e.value.message == "Client error" + + @pytest.mark.usefixtures("disable_gzip") + def test_request_gzip_disabled(self, rsps): + def check_gzip_disabled(request): + assert "Content-Encoding" not in request.headers + assert {"any": "OK"} == json.loads(request.body) + return 200, {}, json.dumps(RESPONSE) + + for method in [rsps.PUT, rsps.POST]: + rsps.add_callback(method, BASE_URL + URL_PATH, check_gzip_disabled) + + API_CLIENT._post(URL_PATH, {"any": "OK"}, headers={}) + API_CLIENT._put(URL_PATH, {"any": "OK"}, headers={}) + + def test_request_gzip_enabled(self, rsps): + def check_gzip_enabled(request): + assert "Content-Encoding" in request.headers + assert {"any": "OK"} == jsgz_load(request.body) + return 200, {}, json.dumps(RESPONSE) + + for method in [rsps.PUT, rsps.POST]: + rsps.add_callback(method, BASE_URL + URL_PATH, check_gzip_enabled) + + API_CLIENT._post(URL_PATH, {"any": "OK"}, headers={}) + API_CLIENT._put(URL_PATH, {"any": "OK"}, headers={}) + + def test_headers_correct(self, mock_all_requests_ok): + API_CLIENT._post(URL_PATH, {"any": "OK"}, headers={"additional": "stuff"}) + headers = mock_all_requests_ok.calls[0].request.headers + + assert "gzip, deflate" == headers["accept-encoding"] + assert "gzip" == headers["content-encoding"] + assert "CognitePythonSDK:{}".format(utils.get_current_sdk_version()) == headers["x-cdp-sdk"] + assert "abc" == headers["api-key"] + assert "stuff" == headers["additional"] + + +class SomeUpdate(CogniteUpdate): + @property + def y(self): + return PrimitiveUpdate(self, "y") + + +class PrimitiveUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> SomeUpdate: + return self._set(value) + + +class SomeResource(CogniteResource): + def __init__(self, x=None, y=None, id=None, external_id=None, cognite_client=None): + self.x = x + self.y = y + self.id = id + self.external_id = external_id + + +class SomeResourceList(CogniteResourceList): + _RESOURCE = SomeResource + _UPDATE = SomeUpdate + + +class SomeFilter(CogniteFilter): + def __init__(self, var_x, var_y): + self.var_x = var_x + self.var_y = var_y + + +class TestStandardRetrieve: + def test_standard_retrieve_OK(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH + "/1", status=200, json={"x": 1, "y": 2}) + assert SomeResource(1, 2) == API_CLIENT._retrieve(cls=SomeResource, resource_path=URL_PATH, id=1) + + def test_standard_retrieve_not_found(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH + "/1", status=404, json={"error": {"message": "Not Found."}}) + assert API_CLIENT._retrieve(cls=SomeResource, resource_path=URL_PATH, id=1) is None + + def test_standard_retrieve_fail(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH + "/1", status=400, json={"error": {"message": "Client Error"}}) + with pytest.raises(CogniteAPIError, match="Client Error") as e: + API_CLIENT._retrieve(cls=SomeResource, resource_path=URL_PATH, id=1) + assert "Client Error" == e.value.message + assert 400 == e.value.code + + def test_cognite_client_is_set(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH + "/1", status=200, json={"x": 1, "y": 2}) + assert COGNITE_CLIENT == API_CLIENT._retrieve(cls=SomeResource, resource_path=URL_PATH, id=1)._cognite_client + + +class TestStandardRetrieveMultiple: + @pytest.fixture + def mock_by_ids(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/byids", status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + yield rsps + + def test_by_id_no_wrap_OK(self, mock_by_ids): + assert SomeResourceList([SomeResource(1, 2), SomeResource(1)]) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=False, ids=[1, 2] + ) + assert {"items": [1, 2]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_single_id_no_wrap_OK(self, mock_by_ids): + assert SomeResource(1, 2) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=False, ids=1 + ) + assert {"items": [1]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_id_wrap_OK(self, mock_by_ids): + assert SomeResourceList([SomeResource(1, 2), SomeResource(1)]) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=[1, 2] + ) + assert {"items": [{"id": 1}, {"id": 2}]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_single_id_wrap_OK(self, mock_by_ids): + assert SomeResource(1, 2) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=1 + ) + assert {"items": [{"id": 1}]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_external_id_wrap_OK(self, mock_by_ids): + assert SomeResourceList([SomeResource(1, 2), SomeResource(1)]) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, external_ids=["1", "2"] + ) + assert {"items": [{"externalId": "1"}, {"externalId": "2"}]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_single_external_id_wrap_OK(self, mock_by_ids): + assert SomeResource(1, 2) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, external_ids="1" + ) + assert {"items": [{"externalId": "1"}]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_by_external_id_no_wrap(self): + with pytest.raises(ValueError, match="must be wrapped"): + API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=False, external_ids=["1", "2"] + ) + + def test_id_and_external_id_mixed(self, mock_by_ids): + assert SomeResourceList([SomeResource(1, 2), SomeResource(1)]) == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=1, external_ids=["2"] + ) + assert {"items": [{"id": 1}, {"externalId": "2"}]} == jsgz_load(mock_by_ids.calls[0].request.body) + + def test_standard_retrieve_multiple_fail(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/byids", status=400, json={"error": {"message": "Client Error"}}) + with pytest.raises(CogniteAPIError, match="Client Error") as e: + API_CLIENT._retrieve_multiple(cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=[1, 2]) + assert "Client Error" == e.value.message + assert 400 == e.value.code + + def test_ids_all_None(self): + with pytest.raises(ValueError, match="No ids specified"): + API_CLIENT._retrieve_multiple(cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=False) + + def test_single_id_not_found(self, rsps): + rsps.add( + rsps.POST, + BASE_URL + URL_PATH + "/byids", + status=400, + json={"error": {"message": "Not Found", "missing": [{"id": 1}]}}, + ) + res = API_CLIENT._retrieve_multiple(cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=1) + assert res is None + + def test_multiple_ids_not_found(self, rsps): + rsps.add( + rsps.POST, + BASE_URL + URL_PATH + "/byids", + status=400, + json={"error": {"message": "Not Found", "missing": [{"id": 1}]}}, + ) + with pytest.raises(CogniteNotFoundError) as e: + API_CLIENT._retrieve_multiple(cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=[1, 2]) + assert e.value.not_found == [{"id": 1}] + + def test_cognite_client_is_set(self, mock_by_ids): + assert ( + COGNITE_CLIENT + == API_CLIENT._retrieve_multiple( + cls=SomeResourceList, resource_path=URL_PATH, wrap_ids=True, ids=[1, 2] + )._cognite_client + ) + + def test_over_limit_concurrent(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/byids", status=200, json={"items": [{"x": 1, "y": 2}]}) + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/byids", status=200, json={"items": [{"x": 3, "y": 4}]}) + + with set_request_limit(API_CLIENT, 1): + API_CLIENT._retrieve_multiple(cls=SomeResourceList, resource_path=URL_PATH, ids=[1, 2], wrap_ids=False) + + assert {"items": [1]} == jsgz_load(rsps.calls[0].request.body) + assert {"items": [2]} == jsgz_load(rsps.calls[1].request.body) + + +class TestStandardList: + def test_standard_list_ok(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + assert ( + SomeResourceList([SomeResource(1, 2), SomeResource(1)]).dump() + == API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="GET").dump() + ) + + def test_standard_list_with_filter_GET_ok(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + assert ( + SomeResourceList([SomeResource(1, 2), SomeResource(1)]).dump() + == API_CLIENT._list( + cls=SomeResourceList, resource_path=URL_PATH, method="GET", filter={"filter": "bla"} + ).dump() + ) + assert "filter=bla" in rsps.calls[0].request.path_url + + def test_standard_list_with_filter_POST_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/list", status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + assert SomeResourceList([SomeResource(1, 2), SomeResource(1)]) == API_CLIENT._list( + cls=SomeResourceList, resource_path=URL_PATH, method="POST", filter={"filter": "bla"} + ) + assert {"filter": {"filter": "bla"}, "limit": 1000, "cursor": None} == jsgz_load(rsps.calls[0].request.body) + + def test_standard_list_fail(self, rsps): + rsps.add(rsps.GET, BASE_URL + URL_PATH, status=400, json={"error": {"message": "Client Error"}}) + with pytest.raises(CogniteAPIError, match="Client Error") as e: + API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="GET") + assert 400 == e.value.code + assert "Client Error" == e.value.message + + NUMBER_OF_ITEMS_FOR_AUTOPAGING = 11500 + ITEMS_TO_GET_WHILE_AUTOPAGING = [{"x": 1, "y": 1} for _ in range(NUMBER_OF_ITEMS_FOR_AUTOPAGING)] + + @pytest.fixture + def mock_get_for_autopaging(self, rsps): + def callback(request): + params = {elem.split("=")[0]: elem.split("=")[1] for elem in request.path_url.split("?")[-1].split("&")} + limit = int(params["limit"]) + cursor = int(params.get("cursor") or 0) + items = self.ITEMS_TO_GET_WHILE_AUTOPAGING[cursor : cursor + limit] + if cursor + limit >= self.NUMBER_OF_ITEMS_FOR_AUTOPAGING: + next_cursor = None + else: + next_cursor = cursor + limit + response = json.dumps({"nextCursor": next_cursor, "items": items}) + return 200, {}, response + + rsps.add_callback(rsps.GET, BASE_URL + URL_PATH, callback) + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_generator(self): + total_resources = 0 + for resource in API_CLIENT._list_generator(cls=SomeResourceList, resource_path=URL_PATH, method="GET"): + assert isinstance(resource, SomeResource) + total_resources += 1 + assert 11500 == total_resources + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_generator_with_limit(self): + total_resources = 0 + for resource in API_CLIENT._list_generator( + cls=SomeResourceList, resource_path=URL_PATH, method="GET", limit=10000 + ): + assert isinstance(resource, SomeResource) + total_resources += 1 + assert 10000 == total_resources + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_generator_with_chunk_size(self): + total_resources = 0 + for resource_chunk in API_CLIENT._list_generator( + cls=SomeResourceList, resource_path=URL_PATH, method="GET", chunk_size=1000 + ): + assert isinstance(resource_chunk, SomeResourceList) + if len(resource_chunk) == 1000: + total_resources += 1000 + elif len(resource_chunk) == 500: + total_resources += 500 + else: + raise AssertionError("resource chunk length was not 1000 or 500") + assert 11500 == total_resources + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_generator_with_chunk_size_with_limit(self): + total_resources = 0 + for resource_chunk in API_CLIENT._list_generator( + cls=SomeResourceList, resource_path=URL_PATH, method="GET", limit=10000, chunk_size=1000 + ): + assert isinstance(resource_chunk, SomeResourceList) + assert 1000 == len(resource_chunk) + total_resources += 1000 + assert 10000 == total_resources + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_generator__chunk_size_exceeds_max(self): + total_resources = 0 + for resource_chunk in API_CLIENT._list_generator( + cls=SomeResourceList, resource_path=URL_PATH, method="GET", limit=2002, chunk_size=1001 + ): + assert isinstance(resource_chunk, SomeResourceList) + assert 1001 == len(resource_chunk) + total_resources += 1001 + assert 2002 == total_resources + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_autopaging(self): + res = API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="GET") + assert self.NUMBER_OF_ITEMS_FOR_AUTOPAGING == len(res) + + @pytest.mark.usefixtures("mock_get_for_autopaging") + def test_standard_list_autopaging_with_limit(self): + res = API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="GET", limit=5333) + assert 5333 == len(res) + + def test_cognite_client_is_set(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/list", status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + rsps.add(rsps.GET, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + assert ( + COGNITE_CLIENT + == API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="POST")._cognite_client + ) + assert ( + COGNITE_CLIENT + == API_CLIENT._list(cls=SomeResourceList, resource_path=URL_PATH, method="GET")._cognite_client + ) + + +class TestStandardCreate: + def test_standard_create_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}, {"x": 1}]}) + res = API_CLIENT._create_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(1, 1), SomeResource(1)] + ) + assert {"items": [{"x": 1, "y": 1}, {"x": 1}]} == jsgz_load(rsps.calls[0].request.body) + assert SomeResource(1, 2) == res[0] + assert SomeResource(1) == res[1] + + def test_standard_create_single_item_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}]}) + res = API_CLIENT._create_multiple(cls=SomeResourceList, resource_path=URL_PATH, items=SomeResource(1, 2)) + assert {"items": [{"x": 1, "y": 2}]} == jsgz_load(rsps.calls[0].request.body) + assert SomeResource(1, 2) == res + + def test_standard_create_single_item_in_list_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}]}) + res = API_CLIENT._create_multiple(cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(1, 2)]) + assert {"items": [{"x": 1, "y": 2}]} == jsgz_load(rsps.calls[0].request.body) + assert SomeResourceList([SomeResource(1, 2)]) == res + + def test_standard_create_fail(self, rsps): + def callback(request): + item = jsgz_load(request.body)["items"][0] + return int(item["externalId"]), {}, json.dumps({}) + + rsps.add_callback(rsps.POST, BASE_URL + URL_PATH, callback=callback, content_type="application/json") + with set_request_limit(API_CLIENT, 1): + with pytest.raises(CogniteAPIError) as e: + API_CLIENT._create_multiple( + cls=SomeResourceList, + resource_path=URL_PATH, + items=[ + SomeResource(1, 1, external_id="200"), + SomeResource(1, external_id="400"), + SomeResource(external_id="500"), + ], + ) + assert 400 == e.value.code + assert [SomeResource(1, external_id="400")] == e.value.failed + assert [SomeResource(1, 1, external_id="200")] == e.value.successful + assert [SomeResource(external_id="500")] == e.value.unknown + + def test_standard_create_concurrent(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}]}) + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 3, "y": 4}]}) + + res = API_CLIENT._create_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(1, 2), SomeResource(3, 4)], limit=1 + ) + assert SomeResourceList([SomeResource(1, 2), SomeResource(3, 4)]) == res + + assert {"items": [{"x": 1, "y": 2}]} == jsgz_load(rsps.calls[0].request.body) + assert {"items": [{"x": 3, "y": 4}]} == jsgz_load(rsps.calls[1].request.body) + + def test_cognite_client_is_set(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH, status=200, json={"items": [{"x": 1, "y": 2}]}) + assert ( + COGNITE_CLIENT + == API_CLIENT._create_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=SomeResource() + )._cognite_client + ) + assert ( + COGNITE_CLIENT + == API_CLIENT._create_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource()] + )._cognite_client + ) + + +class TestStandardDelete: + def test_standard_delete_multiple_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=200, json={}) + API_CLIENT._delete_multiple(resource_path=URL_PATH, wrap_ids=False, ids=[1, 2]) + assert {"items": [1, 2]} == jsgz_load(rsps.calls[0].request.body) + + def test_standard_delete_multiple_ok__single_id(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=200, json={}) + API_CLIENT._delete_multiple(resource_path=URL_PATH, wrap_ids=False, ids=1) + assert {"items": [1]} == jsgz_load(rsps.calls[0].request.body) + + def test_standard_delete_multiple_ok__single_id_in_list(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=200, json={}) + API_CLIENT._delete_multiple(resource_path=URL_PATH, wrap_ids=False, ids=[1]) + assert {"items": [1]} == jsgz_load(rsps.calls[0].request.body) + + def test_standard_delete_multiple_fail_4xx(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=400, json={"error": {"message": "Client Error"}}) + with pytest.raises(CogniteAPIError) as e: + API_CLIENT._delete_multiple(resource_path=URL_PATH, wrap_ids=False, ids=[1, 2]) + assert 400 == e.value.code + assert "Client Error" == e.value.message + assert e.value.failed == [1, 2] + + def test_standard_delete_multiple_fail_5xx(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=500, json={"error": {"message": "Server Error"}}) + with pytest.raises(CogniteAPIError) as e: + API_CLIENT._delete_multiple(resource_path=URL_PATH, wrap_ids=False, ids=[1, 2]) + assert 500 == e.value.code + assert "Server Error" == e.value.message + assert e.value.unknown == [1, 2] + assert e.value.failed == [] + + def test_over_limit_concurrent(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=200, json={}) + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/delete", status=200, json={}) + + with set_request_limit(API_CLIENT, 2): + API_CLIENT._delete_multiple(resource_path=URL_PATH, ids=[1, 2, 3, 4], wrap_ids=False) + assert {"items": [1, 2]} == jsgz_load(rsps.calls[0].request.body) + assert {"items": [3, 4]} == jsgz_load(rsps.calls[1].request.body) + + +class TestStandardUpdate: + @pytest.fixture + def mock_update(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/update", status=200, json={"items": [{"id": 1, "x": 1, "y": 100}]}) + yield rsps + + def test_standard_update_with_cognite_resource_OK(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(id=1, y=100)] + ) + assert SomeResourceList([SomeResource(id=1, x=1, y=100)]) == res + assert {"items": [{"id": 1, "update": {"y": {"set": 100}}}]} == jsgz_load(mock_update.calls[0].request.body) + + def test_standard_update_with_cognite_resource__non_update_attributes(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(id=1, y=100, x=1)] + ) + assert SomeResourceList([SomeResource(id=1, x=1, y=100)]) == res + assert {"items": [{"id": 1, "update": {"y": {"set": 100}}}]} == jsgz_load(mock_update.calls[0].request.body) + + def test_standard_update_with_cognite_resource__id_and_external_id_set(self): + with pytest.raises(AssertionError, match="Exactly one of id and external id"): + API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(id=1, external_id="1", y=100, x=1)] + ) + + def test_standard_update_with_cognite_resource_and_external_id_OK(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(external_id="1", y=100)] + ) + assert SomeResourceList([SomeResource(id=1, x=1, y=100)]) == res + assert {"items": [{"externalId": "1", "update": {"y": {"set": 100}}}]} == jsgz_load( + mock_update.calls[0].request.body + ) + + def test_standard_update_with_cognite_resource__id_error(self): + with pytest.raises(AssertionError, match="one of id and external id"): + API_CLIENT._update_multiple(cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(y=100)]) + + with pytest.raises(AssertionError, match="one of id and external id"): + API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(id=1, external_id="1", y=100)] + ) + + def test_standard_update_with_cognite_update_object_OK(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeUpdate(id=1).y.set(100)] + ) + assert SomeResourceList([SomeResource(id=1, x=1, y=100)]) == res + assert {"items": [{"id": 1, "update": {"y": {"set": 100}}}]} == jsgz_load(mock_update.calls[0].request.body) + + def test_standard_update_single_object(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=SomeUpdate(id=1).y.set(100) + ) + assert SomeResource(id=1, x=1, y=100) == res + assert {"items": [{"id": 1, "update": {"y": {"set": 100}}}]} == jsgz_load(mock_update.calls[0].request.body) + + def test_standard_update_with_cognite_update_object_and_external_id_OK(self, mock_update): + res = API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeUpdate(external_id="1").y.set(100)] + ) + assert SomeResourceList([SomeResource(id=1, x=1, y=100)]) == res + assert {"items": [{"externalId": "1", "update": {"y": {"set": 100}}}]} == jsgz_load( + mock_update.calls[0].request.body + ) + + def test_standard_update_fail_4xx(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/update", status=400, json={"error": {"message": "Client Error"}}) + with pytest.raises(CogniteAPIError) as e: + API_CLIENT._update_multiple( + cls=SomeResourceList, + resource_path=URL_PATH, + items=[SomeResource(id=0), SomeResource(external_id="abc")], + ) + assert e.value.message == "Client Error" + assert e.value.code == 400 + assert e.value.failed == [0, "abc"] + + def test_standard_update_fail_5xx(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/update", status=500, json={"error": {"message": "Server Error"}}) + with pytest.raises(CogniteAPIError) as e: + API_CLIENT._update_multiple( + cls=SomeResourceList, + resource_path=URL_PATH, + items=[SomeResource(id=0), SomeResource(external_id="abc")], + ) + assert e.value.message == "Server Error" + assert e.value.code == 500 + assert e.value.failed == [] + assert e.value.unknown == [0, "abc"] + + def test_cognite_client_is_set(self, mock_update): + assert ( + COGNITE_CLIENT + == API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=SomeResource(id=0) + )._cognite_client + ) + assert ( + COGNITE_CLIENT + == API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(id=0)] + )._cognite_client + ) + + def test_over_limit_concurrent(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/update", status=200, json={"items": [{"x": 1, "y": 2}]}) + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/update", status=200, json={"items": [{"x": 3, "y": 4}]}) + + with set_request_limit(API_CLIENT, 1): + API_CLIENT._update_multiple( + cls=SomeResourceList, resource_path=URL_PATH, items=[SomeResource(1, 2, id=1), SomeResource(3, 4, id=2)] + ) + + assert {"items": [{"id": 1, "update": {"y": {"set": 2}}}]} == jsgz_load(rsps.calls[0].request.body) + assert {"items": [{"id": 2, "update": {"y": {"set": 4}}}]} == jsgz_load(rsps.calls[1].request.body) + + +class TestStandardSearch: + def test_standard_search_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/search", status=200, json={"items": [{"x": 1, "y": 2}]}) + + res = API_CLIENT._search( + cls=SomeResourceList, + resource_path=URL_PATH, + search={"name": "bla"}, + filter=SomeFilter(var_x=1, var_y=1), + limit=1000, + ) + assert SomeResourceList([SomeResource(1, 2)]) == res + assert {"search": {"name": "bla"}, "limit": 1000, "filter": {"varX": 1, "varY": 1}} == jsgz_load( + rsps.calls[0].request.body + ) + + def test_standard_search_dict_filter_ok(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/search", status=200, json={"items": [{"x": 1, "y": 2}]}) + + res = API_CLIENT._search( + cls=SomeResourceList, + resource_path=URL_PATH, + search={"name": "bla"}, + filter={"var_x": 1, "varY": 1}, + limit=1000, + ) + assert SomeResourceList([SomeResource(1, 2)]) == res + assert {"search": {"name": "bla"}, "limit": 1000, "filter": {"varX": 1, "varY": 1}} == jsgz_load( + rsps.calls[0].request.body + ) + + def test_standard_search_fail(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/search", status=400, json={"error": {"message": "Client Error"}}) + + with pytest.raises(CogniteAPIError, match="Client Error") as e: + API_CLIENT._search(cls=SomeResourceList, resource_path=URL_PATH, search=None, filter=None, limit=None) + assert "Client Error" == e.value.message + assert 400 == e.value.code + + def test_cognite_client_is_set(self, rsps): + rsps.add(rsps.POST, BASE_URL + URL_PATH + "/search", status=200, json={"items": [{"x": 1, "y": 2}]}) + + assert ( + COGNITE_CLIENT + == API_CLIENT._search( + cls=SomeResourceList, resource_path=URL_PATH, search={"name": "bla"}, filter={"name": "bla"}, limit=1000 + )._cognite_client + ) + + +class TestHelpers: + @pytest.mark.parametrize( + "input, emulator_url, expected", + [ + ( + "http://localtest.com/api/1.0/projects/test-project/analytics/models", + "http://localhost:8000/api/0.1", + "http://localhost:8000/api/0.1/projects/test-project/models", + ), + ( + "http://localtest.com/api/1.0/projects/test-project/analytics/models/sourcepackages/1", + "http://localhost:1234/api/0.5", + "http://localhost:1234/api/0.5/projects/test-project/models/sourcepackages/1", + ), + ( + "https://api.cognitedata.com/api/0.6/projects/test-project/assets/update", + "http://localhost:8000/api/0.1", + "https://api.cognitedata.com/api/0.6/projects/test-project/assets/update", + ), + ( + "https://api.cognitedata.com/login/status", + "http://localhost:8000/api/0.1", + "https://api.cognitedata.com/login/status", + ), + ], + ) + def test_nostromo_emulator_url_filter(self, input, emulator_url, expected): + os.environ["MODEL_HOSTING_EMULATOR_URL"] = emulator_url + assert expected == API_CLIENT._apply_model_hosting_emulator_url_filter(input) + del os.environ["MODEL_HOSTING_EMULATOR_URL"] + + @pytest.fixture + def mlh_emulator_mock(self, rsps): + os.environ["MODEL_HOSTING_EMULATOR_URL"] = "http://localhost:8888/api/0.1" + rsps.add(rsps.POST, "http://localhost:8888/api/0.1/projects/test-project/models/versions", status=200, json={}) + yield rsps + del os.environ["MODEL_HOSTING_EMULATOR_URL"] + + @pytest.mark.usefixtures("mlh_emulator_mock") + def test_do_request_with_mlh_emulator_activated(self): + API_CLIENT._do_request(method="POST", url_path="/analytics/models/versions") + + @pytest.mark.parametrize( + "ids, external_ids, wrap_ids, expected", + [ + (1, None, False, [1]), + ([1, 2], None, False, [1, 2]), + (1, None, True, [{"id": 1}]), + ([1, 2], None, True, [{"id": 1}, {"id": 2}]), + (1, "1", True, [{"id": 1}, {"externalId": "1"}]), + (1, ["1"], True, [{"id": 1}, {"externalId": "1"}]), + ([1, 2], ["1"], True, [{"id": 1}, {"id": 2}, {"externalId": "1"}]), + (None, "1", True, [{"externalId": "1"}]), + (None, ["1", "2"], True, [{"externalId": "1"}, {"externalId": "2"}]), + ], + ) + def test_process_ids(self, ids, external_ids, wrap_ids, expected): + assert expected == API_CLIENT._process_ids(ids, external_ids, wrap_ids) + + @pytest.mark.parametrize( + "ids, external_ids, wrap_ids, exception, match", + [ + (None, None, False, ValueError, "No ids specified"), + (None, ["1", "2"], False, ValueError, "externalIds must be wrapped"), + ([1], ["1"], False, ValueError, "externalIds must be wrapped"), + ("1", None, False, TypeError, "must be int or list of int"), + (1, 1, True, TypeError, "must be str or list of str"), + ], + ) + def test_process_ids_fail(self, ids, external_ids, wrap_ids, exception, match): + with pytest.raises(exception, match=match): + API_CLIENT._process_ids(ids, external_ids, wrap_ids) + + @pytest.mark.parametrize( + "id, external_id, expected", + [(1, None, True), (None, "1", True), (None, None, False), ([1], None, False), (None, ["1"], False)], + ) + def test_is_single_identifier(self, id, external_id, expected): + assert expected == API_CLIENT._is_single_identifier(id, external_id) + + @pytest.mark.parametrize( + "method, path, expected", + [ + ("GET", "https://api.cognitedata.com/login/status", True), + ("GET", "https://greenfield.cognitedata.com/api/v1/projects/blabla/assets", True), + ("POST", "https://localhost:8000/api/v1/projects/blabla/files/list", True), + ("PUT", "https://api.cognitedata.com/bla", False), + ("POST", "https://greenfield.cognitedata.com/api/v1/projects/blabla/assets", False), + ("PUT", "https://localhost:8000.com/api/v1/projects/blabla/assets", False), + ], + ) + def test_is_retryable(self, method, path, expected): + assert expected == API_CLIENT._is_retryable(method, path) + + @pytest.mark.parametrize( + "method, path", [("POST", "htt://bla/bla"), ("BLOP", "http://localhost:8000/login/status")] + ) + def test_is_retryable_fail(self, method, path): + with pytest.raises(ValueError, match="is not valid"): + API_CLIENT._is_retryable(method, path) + + def test_get_status_codes_to_retry(self): + os.environ["COGNITE_STATUS_FORCELIST"] = "1,2, 3,4" + assert [1, 2, 3, 4] == _get_status_codes_to_retry() + del os.environ["COGNITE_STATUS_FORCELIST"] diff --git a/tests/tests_unit/test_base.py b/tests/tests_unit/test_base.py new file mode 100644 index 0000000000..e86a8fc372 --- /dev/null +++ b/tests/tests_unit/test_base.py @@ -0,0 +1,372 @@ +from unittest import mock + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes._base import * +from cognite.client.exceptions import CogniteMissingClientError + + +class MyResource(CogniteResource): + def __init__(self, var_a=None, var_b=None, id=None, external_id=None, cognite_client=None): + self.var_a = var_a + self.var_b = var_b + self.id = id + self.external_id = external_id + self._cognite_client = cognite_client + + def use(self): + return self._cognite_client + + +class MyUpdate(CogniteUpdate): + @property + def string(self): + return PrimitiveUpdate(self, "string") + + @property + def list(self): + return ListUpdate(self, "list") + + @property + def object(self): + return ObjectUpdate(self, "object") + + +class PrimitiveUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> MyUpdate: + return self._set(value) + + +class ObjectUpdate(CogniteObjectUpdate): + def set(self, value: Dict) -> MyUpdate: + return self._set(value) + + def add(self, value: Dict) -> MyUpdate: + return self._add(value) + + def remove(self, value: List) -> MyUpdate: + return self._remove(value) + + +class ListUpdate(CogniteListUpdate): + def set(self, value: List) -> MyUpdate: + return self._set(value) + + def add(self, value: List) -> MyUpdate: + return self._add(value) + + def remove(self, value: List) -> MyUpdate: + return self._remove(value) + + +class MyFilter(CogniteFilter): + def __init__(self, var_a=None, var_b=None, cognite_client=None): + self.var_a = var_a + self.var_b = var_b + self._cognite_client = cognite_client + + def use(self): + return self._cognite_client + + +class MyResourceList(CogniteResourceList): + _RESOURCE = MyResource + _UPDATE = MyUpdate + + def use(self): + return self._cognite_client + + +class MyResponse(CogniteResponse): + def __init__(self, var_a=None, cognite_client=None): + self.var_a = var_a + self._cognite_client = cognite_client + + @classmethod + def _load(cls, api_response): + data = api_response["data"] + return cls(data["varA"]) + + def use(self): + return self._cognite_client + + +class TestCogniteResource: + def test_dump(self): + assert {"var_a": 1} == MyResource(1).dump() + assert {"var_a": 1} == MyResource(1).dump(camel_case=False) + + def test_dump_camel_case(self): + assert {"varA": 1} == MyResource(1).dump(camel_case=True) + + def test_load(self): + assert MyResource(1).dump() == MyResource._load({"varA": 1}).dump() + assert MyResource(1, 2).dump() == MyResource._load({"var_a": 1, "var_b": 2}).dump() + with pytest.raises(AttributeError, match="'var_c' does not exist"): + MyResource._load({"var_a": 1, "var_c": 1}).dump() + + def test_load_unknown_attribute(self): + with pytest.raises(AttributeError, match="var_c"): + MyResource._load({"varA": 1, "varB": 2, "varC": 3}) + + def test_load_object_attr(self): + assert {"var_a": 1, "var_b": {"camelCase": 1}} == MyResource._load({"varA": 1, "varB": {"camelCase": 1}}).dump() + + def test_eq(self): + assert MyResource(1, "s") == MyResource(1, "s") + assert MyResource(1, "s") == MyResource(1, "s", cognite_client=mock.MagicMock()) + assert MyResource() == MyResource() + assert MyResource(1, "s") != MyResource(1) + assert MyResource(1, "s") != MyResource(2, "t") + + def test_repr(self): + assert json.dumps({"var_a": 1}, indent=4) == MyResource(1).__repr__() + + def test_str_repr(self): + assert json.dumps({"var_a": 1}, indent=4) == MyResource(1).__str__() + + @pytest.mark.dsl + def test_to_pandas(self): + import pandas as pd + + class SomeResource(CogniteResource): + def __init__(self, a_list, ob, ob_expand, ob_ignore, prim, prim_ignore): + self.a_list = a_list + self.ob = ob + self.ob_expand = ob_expand + self.ob_ignore = ob_ignore + self.prim = prim + self.prim_ignore = prim_ignore + + expected_df = pd.DataFrame(columns=["value"]) + expected_df.loc["prim"] = ["abc"] + expected_df.loc["aList"] = [[1, 2, 3]] + expected_df.loc["ob"] = [{"x": "y"}] + expected_df.loc["md_key"] = ["md_value"] + + actual_df = SomeResource([1, 2, 3], {"x": "y"}, {"md_key": "md_value"}, {"bla": "bla"}, "abc", 1).to_pandas( + expand=["obExpand"], ignore=["primIgnore", "obIgnore"] + ) + pd.testing.assert_frame_equal(expected_df, actual_df, check_like=True) + + def test_resource_client_correct(self): + c = CogniteClient() + with pytest.raises(CogniteMissingClientError): + MyResource(1)._cognite_client + assert MyResource(1, cognite_client=c)._cognite_client == c + + def test_use_method_which_requires_cognite_client__client_not_set(self): + mr = MyResource() + with pytest.raises(CogniteMissingClientError): + mr.use() + + +class TestCogniteResourceList: + def test_dump(self): + assert [{"var_a": 1, "var_b": 2}, {"var_a": 2, "var_b": 3}] == MyResourceList( + [MyResource(1, 2), MyResource(2, 3)] + ).dump() + + @pytest.mark.dsl + def test_to_pandas(self): + import pandas as pd + + resource_list = MyResourceList([MyResource(1), MyResource(2, 3)]) + expected_df = pd.DataFrame({"varA": [1, 2], "varB": [None, 3]}) + pd.testing.assert_frame_equal(resource_list.to_pandas(), expected_df) + + def test_load(self): + resource_list = MyResourceList._load([{"varA": 1, "varB": 2}, {"varA": 2, "varB": 3}, {"varA": 3}]) + + assert {"var_a": 1, "var_b": 2} == resource_list[0].dump() + assert [{"var_a": 1, "var_b": 2}, {"var_a": 2, "var_b": 3}, {"var_a": 3}] == resource_list.dump() + + def test_load_unknown_attribute(self): + with pytest.raises(AttributeError, match="var_c"): + MyResourceList._load([{"varA": 1, "varB": 2, "varC": 3}]) + + def test_indexing(self): + resource_list = MyResourceList([MyResource(1, 2), MyResource(2, 3)]) + assert MyResource(1, 2) == resource_list[0] + assert MyResource(2, 3) == resource_list[1] + assert MyResourceList([MyResource(1, 2), MyResource(2, 3)]) == resource_list[:] + assert isinstance(resource_list[:], MyResourceList) + + def test_slice_list_client_remains(self): + mock_client = mock.MagicMock() + rl = MyResourceList([MyResource(1, 2)], cognite_client=mock_client) + rl_sliced = rl[:] + assert rl._cognite_client == rl_sliced._cognite_client + + def test_extend(self): + resource_list = MyResourceList([MyResource(1, 2), MyResource(2, 3)]) + another_resource_list = MyResourceList([MyResource(4, 5), MyResource(6, 7)]) + resource_list.extend(another_resource_list) + assert MyResourceList([MyResource(1, 2), MyResource(2, 3), MyResource(4, 5), MyResource(6, 7)]) == resource_list + + def test_len(self): + resource_list = MyResourceList([MyResource(1, 2), MyResource(2, 3)]) + assert 2 == len(resource_list) + + def test_eq(self): + assert MyResourceList([MyResource(1, 2), MyResource(2, 3)]) == MyResourceList( + [MyResource(1, 2), MyResource(2, 3)] + ) + assert MyResourceList([MyResource(1, 2), MyResource(2, 3)]) != MyResourceList( + [MyResource(2, 3), MyResource(1, 2)] + ) + assert MyResourceList([MyResource(1, 2), MyResource(2, 3)]) != MyResourceList( + [MyResource(2, 3), MyResource(1, 4)] + ) + assert MyResourceList([MyResource(1, 2), MyResource(2, 3)]) != MyResourceList([MyResource(1, 2)]) + + def test_iter(self): + resource_list = MyResourceList([MyResource(1, 2), MyResource(2, 3)]) + counter = 0 + for resource in resource_list: + counter += 1 + assert resource in [MyResource(1, 2), MyResource(2, 3)] + assert 2 == counter + + def test_get_item_by_id(self): + resource_list = MyResourceList([MyResource(id=1, external_id="1"), MyResource(id=2, external_id="2")]) + assert MyResource(id=1, external_id="1") == resource_list.get(id=1) + assert MyResource(id=2, external_id="2") == resource_list.get(id=2) + + def test_get_item_by_external_id(self): + resource_list = MyResourceList([MyResource(id=1, external_id="1"), MyResource(id=2, external_id="2")]) + assert MyResource(id=1, external_id="1") == resource_list.get(external_id="1") + assert MyResource(id=2, external_id="2") == resource_list.get(external_id="2") + + def test_constructor_bad_type(self): + with pytest.raises(TypeError, match="must be of type 'MyResource'"): + MyResourceList([1, 2, 3]) + + def test_resource_list_client_correct(self): + c = CogniteClient() + with pytest.raises(CogniteMissingClientError): + MyResource(1)._cognite_client + assert MyResource(1, cognite_client=c)._cognite_client == c + + def test_use_method_which_requires_cognite_client__client_not_set(self): + mr = MyResourceList([]) + with pytest.raises(CogniteMissingClientError): + mr.use() + + +class TestCogniteFilter: + def test_dump(self): + assert MyFilter(1, 2).dump() == {"var_a": 1, "var_b": 2} + assert MyFilter(1, 2).dump(camel_case=True) == {"varA": 1, "varB": 2} + + def test_eq(self): + assert MyFilter(1, 2) == MyFilter(1, 2) + assert MyFilter(1, 2) == MyFilter(1, 2, cognite_client=mock.MagicMock()) + assert MyFilter(1) != MyFilter(1, 2) + assert MyFilter() == MyFilter() + + def test_str(self): + assert json.dumps({"var_a": 1}, indent=4) == MyFilter(1).__str__() + + def test_repr(self): + assert json.dumps({"var_a": 1}, indent=4) == MyFilter(1).__repr__() + + def test_use_method_which_requires_cognite_client__client_not_set(self): + mr = MyFilter() + with pytest.raises(CogniteMissingClientError): + mr.use() + + +class TestCogniteUpdate: + def test_dump_id(self): + assert {"id": 1, "update": {}} == MyUpdate(id=1).dump() + + def test_dump_external_id(self): + assert {"externalId": "1", "update": {}} == MyUpdate(external_id="1").dump() + + def test_dump_both_ids_set(self): + assert {"id": 1, "update": {}} == MyUpdate(id=1, external_id="1").dump() + + def test_eq(self): + assert MyUpdate() == MyUpdate() + assert MyUpdate(1) == MyUpdate(1) + assert MyUpdate(1).string.set("1") == MyUpdate(1).string.set("1") + assert MyUpdate(1) != MyUpdate(2) + assert MyUpdate(1) != MyUpdate(1).string.set("1") + + def test_str(self): + assert json.dumps(MyUpdate(1).dump(), indent=4) == MyUpdate(1).__str__() + assert json.dumps(MyUpdate(1).string.set("1").dump(), indent=4) == MyUpdate(1).string.set("1").__str__() + + def test_set_string(self): + assert {"id": 1, "update": {"string": {"set": "bla"}}} == MyUpdate(1).string.set("bla").dump() + + def test_add_to_list(self): + assert {"id": 1, "update": {"list": {"add": [1, 2, 3]}}} == MyUpdate(1).list.add([1, 2, 3]).dump() + + def test_set_list(self): + assert {"id": 1, "update": {"list": {"set": [1, 2, 3]}}} == MyUpdate(1).list.set([1, 2, 3]).dump() + + def test_remove_from_list(self): + assert {"id": 1, "update": {"list": {"remove": [1, 2, 3]}}} == MyUpdate(1).list.remove([1, 2, 3]).dump() + + def test_set_object(self): + assert {"id": 1, "update": {"object": {"set": {"key": "value"}}}} == MyUpdate(1).object.set( + {"key": "value"} + ).dump() + + def test_add_object(self): + assert {"id": 1, "update": {"object": {"add": {"key": "value"}}}} == MyUpdate(1).object.add( + {"key": "value"} + ).dump() + + def test_remove_object(self): + assert {"id": 1, "update": {"object": {"remove": ["value"]}}} == MyUpdate(1).object.remove(["value"]).dump() + + def test_set_string_null(self): + assert {"externalId": "1", "update": {"string": {"setNull": True}}} == MyUpdate(external_id="1").string.set( + None + ).dump() + + def test_chain_setters(self): + assert {"id": 1, "update": {"object": {"set": {"bla": "bla"}}, "string": {"set": "bla"}}} == MyUpdate( + id=1 + ).object.set({"bla": "bla"}).string.set("bla").dump() + + def test_get_update_properties(self): + assert {"string", "list", "object"} == set(MyUpdate._get_update_properties()) + + +class TestCogniteResponse: + def test_load(self): + res = MyResponse._load({"data": {"varA": 1}}) + assert 1 == res.var_a + + def test_dump(self): + assert {"var_a": 1} == MyResponse(1).dump() + assert {"varA": 1} == MyResponse(1).dump(camel_case=True) + assert {} == MyResponse().dump() + + def test_str(self): + assert json.dumps(MyResponse(1).dump(), indent=4, sort_keys=True) == MyResponse(1).__str__() + + def test_repr(self): + assert json.dumps(MyResponse(1).dump(), indent=4, sort_keys=True) == MyResponse(1).__repr__() + + def test_eq(self): + assert MyResponse(1) == MyResponse(1) + assert MyResponse(1) == MyResponse(1, cognite_client=mock.MagicMock()) + assert MyResponse(1) != MyResponse(2) + assert MyResponse(1) != MyResponse() + + def test_response_client_correct(self): + c = CogniteClient() + with pytest.raises(CogniteMissingClientError): + MyResource(1)._cognite_client + assert MyResource(1, cognite_client=c)._cognite_client == c + + def test_use_method_which_requires_cognite_client__client_not_set(self): + mr = MyResponse() + with pytest.raises(CogniteMissingClientError): + mr.use() diff --git a/tests/tests_unit/test_cognite_client.py b/tests/tests_unit/test_cognite_client.py new file mode 100644 index 0000000000..b2ff6ea020 --- /dev/null +++ b/tests/tests_unit/test_cognite_client.py @@ -0,0 +1,163 @@ +import logging +import os +import random +import sys +import threading +import types +from multiprocessing.pool import ThreadPool +from time import sleep + +import pytest + +from cognite.client import CogniteClient +from cognite.client._api.assets import AssetList +from cognite.client._api.files import FileMetadataList +from cognite.client._api.time_series import TimeSeriesList +from cognite.client.data_classes import Asset, Event, FileMetadata, TimeSeries +from cognite.client.exceptions import CogniteAPIKeyError +from cognite.client.utils._utils import DebugLogFormatter +from tests.utils import BASE_URL + + +@pytest.fixture +def default_client_config(): + from cognite.client._cognite_client import DEFAULT_MAX_WORKERS, DEFAULT_TIMEOUT + + yield "https://greenfield.cognitedata.com", DEFAULT_MAX_WORKERS, DEFAULT_TIMEOUT, "python-sdk-integration-tests" + + +@pytest.fixture +def environment_client_config(): + base_url = "blabla" + num_of_workers = 1 + timeout = 10 + client_name = "test-client" + + tmp_base_url = os.environ["COGNITE_BASE_URL"] + tmp_client_name = os.environ["COGNITE_CLIENT_NAME"] + os.environ["COGNITE_BASE_URL"] = base_url + os.environ["COGNITE_MAX_WORKERS"] = str(num_of_workers) + os.environ["COGNITE_TIMEOUT"] = str(timeout) + os.environ["COGNITE_CLIENT_NAME"] = client_name + + yield base_url, num_of_workers, timeout, client_name + + os.environ["COGNITE_BASE_URL"] = tmp_base_url + del os.environ["COGNITE_MAX_WORKERS"] + del os.environ["COGNITE_TIMEOUT"] + os.environ["COGNITE_CLIENT_NAME"] = tmp_client_name + + +class TestCogniteClient: + def test_project_is_correct(self, rsps_with_login_mock): + c = CogniteClient() + assert c.project == "test" + + @pytest.fixture + def unset_env_api_key(self): + tmp = os.environ["COGNITE_API_KEY"] + del os.environ["COGNITE_API_KEY"] + yield + os.environ["COGNITE_API_KEY"] = tmp + + def test_no_api_key_set(self, unset_env_api_key): + with pytest.raises(ValueError, match="No API Key has been specified"): + CogniteClient() + + def test_invalid_api_key(self, rsps): + rsps.add( + rsps.GET, + BASE_URL + "/login/status", + status=200, + json={"data": {"project": "", "loggedIn": False, "user": "", "projectId": -1}}, + ) + with pytest.raises(CogniteAPIKeyError): + CogniteClient() + + @pytest.fixture + def unset_env_client_name(self): + tmp = os.environ["COGNITE_CLIENT_NAME"] + del os.environ["COGNITE_CLIENT_NAME"] + yield + os.environ["COGNITE_CLIENT_NAME"] = tmp + + def test_no_client_name(self, unset_env_client_name): + with pytest.raises(ValueError, match="No client name has been specified"): + CogniteClient() + + def assert_config_is_correct(self, client, base_url, max_workers, timeout, client_name): + assert client._base_url == base_url + assert type(client._base_url) is str + + assert client._max_workers == max_workers + assert type(client._max_workers) is int + + assert client._timeout == timeout + assert type(client._timeout) is int + + assert client._client_name == client_name + assert type(client._client_name) is str + + def test_default_config(self, client, default_client_config): + self.assert_config_is_correct(client, *default_client_config) + + def test_parameter_config(self): + base_url = "blabla" + max_workers = 1 + timeout = 10 + client_name = "test-client" + + client = CogniteClient( + project="something", base_url=base_url, max_workers=max_workers, timeout=timeout, client_name=client_name + ) + self.assert_config_is_correct(client, base_url, max_workers, timeout, client_name) + + def test_environment_config(self, environment_client_config): + client = CogniteClient(project="something") + self.assert_config_is_correct(client, *environment_client_config) + + @pytest.fixture + def thread_local_credentials_module(self): + credentials_module = types.ModuleType("cognite._thread_local") + credentials_module.credentials = threading.local() + sys.modules["cognite._thread_local"] = credentials_module + yield + del sys.modules["cognite._thread_local"] + + def create_client_and_check_config(self, i): + from cognite._thread_local import credentials + + api_key = "thread-local-api-key{}".format(i) + project = "thread-local-project{}".format(i) + + credentials.api_key = api_key + credentials.project = project + + sleep(random.random()) + client = CogniteClient() + + assert api_key == client._CogniteClient__api_key + assert project == client.project + + def test_create_client_thread_local_config(self, thread_local_credentials_module): + with ThreadPool() as pool: + pool.map(self.create_client_and_check_config, list(range(16))) + + def test_client_debug_mode(self, rsps_with_login_mock): + CogniteClient(debug=True) + log = logging.getLogger("cognite-sdk") + assert isinstance(log.handlers[0].formatter, DebugLogFormatter) + log.handlers = [] + log.propagate = False + + +class TestInstantiateWithClient: + @pytest.mark.parametrize("cls", [Asset, Event, FileMetadata, TimeSeries]) + def test_instantiate_resources_with_cognite_client(self, cls): + c = CogniteClient() + assert cls(cognite_client=c)._cognite_client == c + + @pytest.mark.parametrize("cls", [AssetList, Event, FileMetadataList, TimeSeriesList]) + def test_intantiate_resource_lists_with_cognite_client(self, cls): + c = CogniteClient() + assert cls([], cognite_client=c)._cognite_client == c diff --git a/tests/tests_unit/test_data_classes/__init__.py b/tests/tests_unit/test_data_classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_data_classes/test_assets.py b/tests/tests_unit/test_data_classes/test_assets.py new file mode 100644 index 0000000000..12a5f9af7f --- /dev/null +++ b/tests/tests_unit/test_data_classes/test_assets.py @@ -0,0 +1,204 @@ +from unittest import mock +from unittest.mock import call + +from cognite.client import CogniteClient +from cognite.client.data_classes import Asset, AssetList + +c = CogniteClient() + + +class TestAsset: + def test_get_events(self): + c.events.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.events() + assert c.events.list.call_args == call(asset_ids=[1]) + assert c.events.list.call_count == 1 + + def test_get_time_series(self): + c.time_series.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.time_series() + assert c.time_series.list.call_args == call(asset_ids=[1]) + assert c.time_series.list.call_count == 1 + + def test_get_files(self): + c.files.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.files() + assert c.files.list.call_args == call(asset_ids=[1]) + assert c.files.list.call_count == 1 + + def test_get_parent(self): + c.assets.retrieve = mock.MagicMock() + a1 = Asset(parent_id=1, cognite_client=c) + a1.parent() + assert c.assets.retrieve.call_args == call(id=1) + assert c.assets.retrieve.call_count == 1 + + def test_get_children(self): + c.assets.list = mock.MagicMock() + a1 = Asset(id=1, cognite_client=c) + a1.children() + assert c.assets.list.call_args == call(parent_ids=[1], limit=None) + assert c.assets.list.call_count == 1 + + def test_get_subtree(self): + c.assets.retrieve_subtree = mock.MagicMock() + a1 = Asset(id=1, cognite_client=c) + a1.subtree(depth=1) + assert c.assets.retrieve_subtree.call_args == call(id=1, depth=1) + assert c.assets.retrieve_subtree.call_count == 1 + + +class TestAssetList: + def test_get_events(self): + c.events.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.events() + assert c.events.list.call_args == call(asset_ids=[1]) + assert c.events.list.call_count == 1 + + def test_get_time_series(self): + c.time_series.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.time_series() + assert c.time_series.list.call_args == call(asset_ids=[1]) + assert c.time_series.list.call_count == 1 + + def test_get_files(self): + c.files.list = mock.MagicMock() + a = Asset(id=1, cognite_client=c) + a.files() + assert c.files.list.call_args == call(asset_ids=[1]) + assert c.files.list.call_count == 1 + + +class TestAssetHierarchyVisualization: + def test_normal_tree(self): + assets = AssetList( + [Asset(id=1, path=[1]), Asset(id=2, path=[1, 2]), Asset(id=3, path=[1, 3]), Asset(id=4, path=[1, 3, 4])] + ) + assert """ +1 +path: [1] +|______ 2 + path: [1, 2] +|______ 3 + path: [1, 3] + |______ 4 + path: [1, 3, 4] +""" == str( + assets + ) + + def test_multiple_root_nodes(self): + assets = AssetList( + [ + Asset(id=1, path=[1]), + Asset(id=2, path=[2]), + Asset(id=3, path=[1, 3]), + Asset(id=4, path=[2, 4]), + Asset(id=5, path=[2, 4, 5]), + ] + ) + assert """ +1 +path: [1] +|______ 3 + path: [1, 3] + +******************************************************************************** + +2 +path: [2] +|______ 4 + path: [2, 4] + |______ 5 + path: [2, 4, 5] +""" == str( + assets + ) + + def test_parent_nodes_missing(self): + assets = AssetList( + [ + Asset(id=1, path=[1]), + Asset(id=2, path=[1, 2]), + Asset(id=4, path=[1, 2, 3, 4]), + Asset(id=6, path=[1, 5, 6]), + ] + ) + assert """ +1 +path: [1] +|______ 2 + path: [1, 2] + +-------------------------------------------------------------------------------- + + |______ 4 + path: [1, 2, 3, 4] + +-------------------------------------------------------------------------------- + + |______ 6 + path: [1, 5, 6] +""" == str( + assets + ) + + def test_expand_dicts(self): + assets = AssetList([Asset(id=1, path=[1], metadata={"a": "b", "c": "d"})]) + assert """ +1 +metadata: + - a: b + - c: d +path: [1] +""" == str( + assets + ) + + def test_all_cases_combined(self): + assets = AssetList( + [ + Asset(id=1, path=[1]), + Asset(id=3, path=[2, 3], metadata={"k1": "v1", "k2": "v2"}), + Asset(id=2, path=[2]), + Asset(id=4, path=[10, 4]), + Asset(id=99, path=[20, 99]), + Asset(id=5, path=[20, 10, 5]), + ] + ) + assert """ +1 +path: [1] + +******************************************************************************** + +2 +path: [2] +|______ 3 + metadata: + - k1: v1 + - k2: v2 + path: [2, 3] + +******************************************************************************** + +|______ 4 + path: [10, 4] + +******************************************************************************** + + |______ 5 + path: [20, 10, 5] + +-------------------------------------------------------------------------------- + +|______ 99 + path: [20, 99] +""" == str( + assets + ) diff --git a/tests/tests_unit/test_docstring_examples.py b/tests/tests_unit/test_docstring_examples.py new file mode 100644 index 0000000000..c5ac649928 --- /dev/null +++ b/tests/tests_unit/test_docstring_examples.py @@ -0,0 +1,37 @@ +import doctest +from unittest import TextTestRunner + +import pytest + +from cognite.client._api import assets, datapoints, events, files, login, raw, time_series + + +def run_docstring_tests(module): + runner = TextTestRunner() + s = runner.run(doctest.DocTestSuite(module)) + assert 0 == len(s.failures) + + +@pytest.mark.usefixtures("mock_cognite_client") +class TestDocstringExamples: + def test_time_series(self): + run_docstring_tests(time_series) + + def test_assets(self): + run_docstring_tests(assets) + + @pytest.mark.dsl + def test_datapoints(self): + run_docstring_tests(datapoints) + + def test_events(self): + run_docstring_tests(events) + + def test_files(self): + run_docstring_tests(files) + + def test_login(self): + run_docstring_tests(login) + + def test_raw(self): + run_docstring_tests(raw) diff --git a/tests/tests_unit/test_exceptions.py b/tests/tests_unit/test_exceptions.py new file mode 100644 index 0000000000..9199403814 --- /dev/null +++ b/tests/tests_unit/test_exceptions.py @@ -0,0 +1,20 @@ +from cognite.client.exceptions import CogniteAPIError + + +class TestAPIError: + def test_api_error(self): + e = CogniteAPIError( + message="bla", + code=200, + x_request_id="abc", + missing=[{"id": 123}], + duplicated=[{"externalId": "abc"}], + successful=["bla"], + ) + assert "bla" == e.message + assert 200 == e.code + assert "abc" == e.x_request_id + assert [{"id": 123}] == e.missing + assert [{"externalId": "abc"}] == e.duplicated + + assert "bla" in e.__str__() diff --git a/tests/tests_unit/test_utils.py b/tests/tests_unit/test_utils.py new file mode 100644 index 0000000000..f808ca48d0 --- /dev/null +++ b/tests/tests_unit/test_utils.py @@ -0,0 +1,278 @@ +import json +from datetime import datetime +from decimal import Decimal +from time import sleep +from unittest import mock + +import pytest + +from cognite.client.exceptions import CogniteImportError +from cognite.client.utils import _utils as utils + + +class TestDatetimeToMs: + def test_datetime_to_ms(self): + from datetime import datetime + + assert utils.datetime_to_ms(datetime(2018, 1, 31)) == 1517356800000 + assert utils.datetime_to_ms(datetime(2018, 1, 31, 11, 11, 11)) == 1517397071000 + assert utils.datetime_to_ms(datetime(100, 1, 31)) == -59008867200000 + with pytest.raises(TypeError): + utils.datetime_to_ms(None) + + def test_ms_to_datetime(self): + from datetime import datetime + + assert utils.ms_to_datetime(1517356800000) == datetime(2018, 1, 31) + assert utils.ms_to_datetime(1517397071000) == datetime(2018, 1, 31, 11, 11, 11) + assert utils.ms_to_datetime(-59008867200000) == datetime(100, 1, 31) + with pytest.raises(TypeError): + utils.ms_to_datetime(None) + + +class TestTimestampToMs: + @pytest.mark.parametrize("t", [None, [], {}]) + def test_invalid_type(self, t): + with pytest.raises(TypeError, match="must be"): + utils.timestamp_to_ms(t) + + def test_ms(self): + assert 1514760000000 == utils.timestamp_to_ms(1514760000000) + assert 1514764800000 == utils.timestamp_to_ms(1514764800000) + + def test_datetime(self): + assert 1514764800000 == utils.timestamp_to_ms(datetime(2018, 1, 1)) + assert 1546300800000 == utils.timestamp_to_ms(datetime(2019, 1, 1)) + + def test_float(self): + assert 1514760000000 == utils.timestamp_to_ms(1514760000000.0) + assert 1514764800000 == utils.timestamp_to_ms(1514764800000.0) + + @mock.patch("cognite.client.utils._utils.time.time") + @pytest.mark.parametrize( + "time_ago_string, expected_timestamp", + [ + ("now", 10 ** 12), + ("1s-ago", 10 ** 12 - 1 * 1000), + ("13s-ago", 10 ** 12 - 13 * 1000), + ("1m-ago", 10 ** 12 - 1 * 60 * 1000), + ("13m-ago", 10 ** 12 - 13 * 60 * 1000), + ("1h-ago", 10 ** 12 - 1 * 60 * 60 * 1000), + ("13h-ago", 10 ** 12 - 13 * 60 * 60 * 1000), + ("1d-ago", 10 ** 12 - 1 * 24 * 60 * 60 * 1000), + ("13d-ago", 10 ** 12 - 13 * 24 * 60 * 60 * 1000), + ("1w-ago", 10 ** 12 - 1 * 7 * 24 * 60 * 60 * 1000), + ("13w-ago", 10 ** 12 - 13 * 7 * 24 * 60 * 60 * 1000), + ], + ) + def test_time_ago(self, time_mock, time_ago_string, expected_timestamp): + time_mock.return_value = 10 ** 9 + + assert utils.timestamp_to_ms(time_ago_string) == expected_timestamp + + @pytest.mark.parametrize("time_ago_string", ["1s", "4h", "13m-ag", "13m ago", "bla"]) + def test_invalid(self, time_ago_string): + with pytest.raises(ValueError, match=time_ago_string): + utils.timestamp_to_ms(time_ago_string) + + def test_time_ago_real_time(self): + expected_time_now = datetime.now().timestamp() * 1000 + time_now = utils.timestamp_to_ms("now") + assert abs(expected_time_now - time_now) < 10 + + sleep(0.2) + + time_now = utils.timestamp_to_ms("now") + assert abs(expected_time_now - time_now) > 190 + + @pytest.mark.parametrize("t", [-1, datetime(1969, 12, 31), "100000000w-ago"]) + def test_negative(self, t): + with pytest.raises(ValueError, match="negative"): + utils.timestamp_to_ms(t) + + +class TestGranularityToMs: + @pytest.mark.parametrize( + "granularity, expected_ms", + [ + ("1s", 1 * 1000), + ("13s", 13 * 1000), + ("1m", 1 * 60 * 1000), + ("13m", 13 * 60 * 1000), + ("1h", 1 * 60 * 60 * 1000), + ("13h", 13 * 60 * 60 * 1000), + ("1d", 1 * 24 * 60 * 60 * 1000), + ("13d", 13 * 24 * 60 * 60 * 1000), + ], + ) + def test_to_ms(self, granularity, expected_ms): + assert utils.granularity_to_ms(granularity) == expected_ms + + @pytest.mark.parametrize("granularity", ["2w", "-3h", "13m-ago", "13", "bla"]) + def test_to_ms_invalid(self, granularity): + with pytest.raises(ValueError, match=granularity): + utils.granularity_to_ms(granularity) + + +class TestGranularityUnitToMs: + @pytest.mark.parametrize( + "granularity, expected_ms", + [ + ("1s", 1 * 1000), + ("13s", 1 * 1000), + ("1m", 1 * 60 * 1000), + ("13m", 1 * 60 * 1000), + ("1h", 1 * 60 * 60 * 1000), + ("13h", 1 * 60 * 60 * 1000), + ("1d", 1 * 24 * 60 * 60 * 1000), + ("13d", 1 * 24 * 60 * 60 * 1000), + ], + ) + def test_to_ms(self, granularity, expected_ms): + assert utils.granularity_unit_to_ms(granularity) == expected_ms + + @pytest.mark.parametrize("granularity", ["2w", "-3h", "13m-ago", "13", "bla"]) + def test_to_ms_invalid(self, granularity): + with pytest.raises(ValueError, match="format"): + utils.granularity_unit_to_ms(granularity) + + +class TestCaseConversion: + def test_to_camel_case(self): + assert "camelCase" == utils.to_camel_case("camel_case") + assert "camelCase" == utils.to_camel_case("camelCase") + assert "a" == utils.to_camel_case("a") + + def test_to_snake_case(self): + assert "snake_case" == utils.to_snake_case("snakeCase") + assert "snake_case" == utils.to_snake_case("snake_case") + assert "a" == utils.to_snake_case("a") + + +class TestLocalImport: + @pytest.mark.dsl + def test_local_import_single_ok(self): + import pandas + + assert pandas == utils.local_import("pandas") + + @pytest.mark.dsl + def test_local_import_multiple_ok(self): + import pandas, numpy + + assert (pandas, numpy) == utils.local_import("pandas", "numpy") + + def test_local_import_single_fail(self): + with pytest.raises(CogniteImportError, match="requires 'not-a-module' to be installed"): + utils.local_import("not-a-module") + + @pytest.mark.dsl + def test_local_import_multiple_fail(self): + with pytest.raises(CogniteImportError, match="requires 'not-a-module' to be installed"): + utils.local_import("pandas", "not-a-module") + + @pytest.mark.coredeps + def test_dsl_deps_not_installed(self): + for dep in ["numpy", "pandas"]: + with pytest.raises(CogniteImportError, match=dep): + utils.local_import(dep) + + +class TestUrlEncode: + def test_url_encode(self): + assert "/bla/yes%2Fno/bla" == utils.interpolate_and_url_encode("/bla/{}/bla", "yes/no") + assert "/bla/123/bla/456" == utils.interpolate_and_url_encode("/bla/{}/bla/{}", "123", "456") + + +class TestJsonDumpDefault: + def test_json_serializable_Decimal(self): + with pytest.raises(TypeError): + json.dumps(Decimal(1)) + + assert json.dumps(Decimal(1), default=utils.json_dump_default) + + def test_json_serializable_object(self): + class Obj: + def __init__(self): + self.x = 1 + + with pytest.raises(TypeError): + json.dumps(Obj()) + + assert json.dumps({"x": 1}) == json.dumps(Obj(), default=utils.json_dump_default) + + @pytest.mark.dsl + def test_json_serialiable_numpy_integer(self): + import numpy as np + + inputs = [np.int32(1), np.int64(1)] + for input in inputs: + assert json.dumps(input, default=utils.json_dump_default) + + +class TestSplitIntoChunks: + @pytest.mark.parametrize( + "input, chunk_size, expected_output", + [ + (["a", "b", "c"], 1, [["a"], ["b"], ["c"]]), + (["a", "b", "c"], 2, [["a", "b"], ["c"]]), + ([], 1000, []), + (["a", "b", "c"], 3, [["a", "b", "c"]]), + (["a", "b", "c"], 10, [["a", "b", "c"]]), + ({"a": 1, "b": 2}, 1, [{"a": 1}, {"b": 2}]), + ({"a": 1, "b": 2}, 2, [{"a": 1, "b": 2}]), + ({"a": 1, "b": 2}, 3, [{"a": 1, "b": 2}]), + ({}, 1, []), + ], + ) + def test_split_into_chunks(self, input, chunk_size, expected_output): + actual_output = utils.split_into_chunks(input, chunk_size) + assert len(actual_output) == len(expected_output) + for element in expected_output: + assert element in actual_output + + +class TestAssertions: + @pytest.mark.parametrize("timestamp", [utils.timestamp_to_ms(datetime(2018, 1, 1)), datetime(1971, 1, 2), "now"]) + def test_assert_timestamp_not_in_1970_ok(self, timestamp): + utils.assert_timestamp_not_in_1970(timestamp) + + @pytest.mark.parametrize("timestamp", [utils.timestamp_to_ms(datetime(2018, 1, 1)) / 1000, datetime(1970, 2, 1)]) + def test_assert_timestamp_not_in_1970_fail(self, timestamp): + with pytest.raises(AssertionError, match="You are attempting to post data in 1970"): + utils.assert_timestamp_not_in_1970(timestamp) + + @pytest.mark.parametrize("var, var_name, types", [(1, "var1", [int]), ("1", "var2", [int, str])]) + def test_assert_type_ok(self, var, var_name, types): + utils.assert_type(var, var_name, types=types) + + @pytest.mark.parametrize("var, var_name, types", [("1", "var", [int, float]), ((1,), "var2", [dict, list])]) + def test_assert_type_fail(self, var, var_name, types): + with pytest.raises(TypeError, match=str(types)): + utils.assert_type(var, var_name, types) + + def test_assert_exactly_one_of_id_and_external_id(self): + with pytest.raises(AssertionError): + utils.assert_exactly_one_of_id_or_external_id(1, "1") + utils.assert_exactly_one_of_id_or_external_id(1, None) + utils.assert_exactly_one_of_id_or_external_id(None, "1") + + +class TestObjectTimeConversion: + @pytest.mark.parametrize( + "item, expected_output", + [ + ({"created_time": 0}, {"created_time": "1970-01-01 00:00:00"}), + ({"last_updated_time": 0}, {"last_updated_time": "1970-01-01 00:00:00"}), + ({"start_time": 0}, {"start_time": "1970-01-01 00:00:00"}), + ({"end_time": 0}, {"end_time": "1970-01-01 00:00:00"}), + ({"not_a_time": 0}, {"not_a_time": 0}), + ([{"created_time": 0}], [{"created_time": "1970-01-01 00:00:00"}]), + ([{"last_updated_time": 0}], [{"last_updated_time": "1970-01-01 00:00:00"}]), + ([{"start_time": 0}], [{"start_time": "1970-01-01 00:00:00"}]), + ([{"end_time": 0}], [{"end_time": "1970-01-01 00:00:00"}]), + ([{"not_a_time": 0}], [{"not_a_time": 0}]), + ], + ) + def test_convert_time_attributes_to_datetime(self, item, expected_output): + assert expected_output == utils.convert_time_attributes_to_datetime(item) diff --git a/tests/utils.py b/tests/utils.py index 0c8f599987..c0de4f92b2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,9 +1,54 @@ +import cProfile +import functools import gzip import json +from contextlib import contextmanager +from unittest import mock +from unittest.mock import PropertyMock +BASE_URL = "https://greenfield.cognitedata.com" -def get_call_args_data_from_mock(mock, index, decompress_gzip=False): - data = mock.call_args_list[index][1]["data"] - if decompress_gzip: - data = gzip.decompress(data).decode() - return json.loads(data) + +def jsgz_load(s): + return json.loads(gzip.decompress(s).decode()) + + +@contextmanager +def profilectx(): + pr = cProfile.Profile() + pr.enable() + yield + pr.disable() + pr.print_stats(sort="cumtime") + + +def profile(method): + @functools.wraps(method) + def wrapper(*args, **kwargs): + with profilectx(): + method(*args, **kwargs) + + return wrapper + + +@contextmanager +def set_request_limit(client, limit): + limits = [ + "_CREATE_LIMIT", + "_LIST_LIMIT", + "_RETRIEVE_LIMIT", + "_UPDATE_LIMIT", + "_DELETE_LIMIT", + "_DPS_LIMIT", + "_DPS_LIMIT_AGG", + ] + + tmp = {l: 0 for l in limits} + for limit_name in limits: + if hasattr(client, limit_name): + tmp[limit_name] = getattr(client, limit_name) + setattr(client, limit_name, limit) + yield + for limit_name, limit_val in tmp.items(): + if hasattr(client, limit_name): + setattr(client, limit_name, limit_val) diff --git a/tox.ini b/tox.ini index 240099ae27..5128bffcf9 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,14 @@ envlist = py35,py36,py37 deps = pytest pytest-cov + pytest-mock + responses + matplotlib commands = - pytest --cov-report xml:coverage.xml --cov=cognite --junitxml=test-report.xml + pytest tests --cov-report xml:coverage.xml --cov=cognite --junitxml=test-report.xml {posargs} setenv = COGNITE_API_KEY={env:COGNITE_API_KEY} + COGNITE_BASE_URL={env:COGNITE_BASE_URL} + COGNITE_CLIENT_NAME={env:COGNITE_CLIENT_NAME}