From 222e6455621e89096deb7f83fd094fbbdcfbeea6 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre Date: Tue, 1 Dec 2020 13:22:01 -0600 Subject: [PATCH] PXP-7251 Fix for GUID prefixes (#18) --- Pipfile | 1 + Pipfile.lock | 269 +++++++++++++---------- manifestservice/manifests/__init__.py | 24 +- tests/broken_s3_test.py | 75 ------- tests/cohorts_test.py | 85 +++++++ tests/conftest.py | 154 ++++++++++++- tests/{app_test.py => manifests_test.py} | 128 +---------- 7 files changed, 402 insertions(+), 334 deletions(-) delete mode 100644 tests/broken_s3_test.py create mode 100644 tests/cohorts_test.py rename tests/{app_test.py => manifests_test.py} (58%) diff --git a/Pipfile b/Pipfile index ceb9b04..6df81a7 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ codacy-coverage = "*" pytest="~=4.3" pytest-mock="~=1.10" pytest-flask="~=0.15.0" +unittest2="~=1.1.0" [packages] flask="~=1.1.2" diff --git a/Pipfile.lock b/Pipfile.lock index e2502d2..f137fc4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8e110abff99701868bc141fb882ade6f0f06a4045cfce3cd8555241348be5d67" + "sha256": "6fb2c2eea9a06d4fbe641c292e5f58fa10850dc0309d42907ba488afb708759f" }, "pipfile-spec": 6, "requires": { @@ -18,10 +18,10 @@ "default": { "addict": { "hashes": [ - "sha256:69904f5650015d93043522b27316649b6dc877ab1eabb9ea9eb7b11386882edc", - "sha256:e8085624d18a9eecd7ec8a427f378f1e5036dfafba34c33fa29f0be8cd3a80b2" + "sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc", + "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494" ], - "version": "==2.3.0" + "version": "==2.4.0" }, "argparse": { "hashes": [ @@ -46,25 +46,25 @@ }, "babel": { "hashes": [ - "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38", - "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4" + "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5", + "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05" ], - "version": "==2.8.0" + "version": "==2.9.0" }, "boto3": { "hashes": [ - "sha256:2e16f02c8b832d401d958d7ca0a14c5bc7da17827918e6b24e5bc43dce8f496e", - "sha256:ab5353a968a4e664b9da2dd950169b755066525fcbfdfc90e7e49c8333d95c19" + "sha256:13e6ab0abe10b4de54583067d12ed48c8477d91b7a7a8e6097d366a2a3631d17", + "sha256:fb25ec578fe59aee298da2c976e2bcfd79a2eabead386e8e4f68bb3eb4ec50b3" ], "index": "pypi", - "version": "==1.16.0" + "version": "==1.16.25" }, "botocore": { "hashes": [ - "sha256:226effa72e3ddd0a802e812c0e204999393ca7982fee754cc0c770a7a1caef3a", - "sha256:9bf8586b69f20cf0a8ed1e27338cd10ce847751d1a2fd98b92662565c8a2df24" + "sha256:4e488351f399501a78b16e1dc466e505ccf63897c936030c56c6d3c37e57939c", + "sha256:ccaf3979590b72625b3699d93dabf48f350f9a3304c127fc6830e8ac842b0d96" ], - "version": "==1.19.0" + "version": "==1.19.25" }, "cached-property": { "hashes": [ @@ -87,51 +87,49 @@ }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" ], - "version": "==2020.6.20" + "version": "==2020.11.8" }, "cffi": { "hashes": [ - "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", - "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", - "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", - "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", - "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", - "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", - "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", - "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", - "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", - "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", - "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", - "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", - "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", - "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", - "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", - "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", - "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", - "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", - "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", - "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", - "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", - "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", - "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", - "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", - "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", - "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", - "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", - "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", - "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", - "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", - "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", - "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", - "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", - "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", - "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", - "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" - ], - "version": "==1.14.3" + "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e", + "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d", + "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a", + "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec", + "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362", + "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668", + "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c", + "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b", + "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06", + "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698", + "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2", + "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c", + "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7", + "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009", + "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03", + "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b", + "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909", + "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53", + "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35", + "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26", + "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b", + "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb", + "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293", + "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd", + "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d", + "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3", + "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d", + "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca", + "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d", + "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775", + "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375", + "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b", + "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b", + "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f" + ], + "version": "==1.14.4" }, "chardet": { "hashes": [ @@ -142,10 +140,10 @@ }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:681c9380a24b22fec089c8e5ffe40aa16a0da79f248a26fe2481bfa8170bfcc1", + "sha256:e4315a188403c0258bbc4a4e31863e48fc301c4e95b8007a8eeda0391158df13" ], - "version": "==7.1.2" + "version": "==8.0.0a1" }, "cryptography": { "hashes": [ @@ -198,19 +196,19 @@ }, "importlib-metadata": { "hashes": [ - "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", - "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" + "sha256:590690d61efdd716ff82c39ca9a9d4209252adfe288a4b5721181050acbd4175", + "sha256:d9b8a46a0885337627a6430db287176970fff18ad421becec1d64cfc763c2099" ], "markers": "python_version < '3.8'", - "version": "==2.0.0" + "version": "==3.1.0" }, "importlib-resources": { "hashes": [ - "sha256:19f745a6eca188b490b1428c8d1d4a0d2368759f32370ea8fb89cad2ab1106c3", - "sha256:d028f66b66c0d5732dae86ba4276999855e162a749c92620a38c1d779ed138a7" + "sha256:7b51f0106c8ec564b1bef3d9c588bc694ce2b92125bbb6278f4f2f5b54ec3592", + "sha256:a3d34a8464ce1d5d7c92b0ea4e921e696d86f2aa212e684451cb1482c8d84ed5" ], "markers": "python_version < '3.7'", - "version": "==3.0.0" + "version": "==3.3.0" }, "iso8601": { "hashes": [ @@ -327,10 +325,10 @@ }, "oslo.config": { "hashes": [ - "sha256:490bc3dc05ad69f77215dc650b1aa0a1bd5e5fb0415e925b6045552b828213aa", - "sha256:c214ab7b30946b7341248edc7ea000944163b460d395af99fd182598b82d5833" + "sha256:829a6dac59be1880e7b3ae510a7e122a43ba4bfb1d90f59ccefc90d8549d82ad", + "sha256:a3b9f9e6e74e40baa46bdb1a94b149087388003f2a51e591cf937e864c951ee0" ], - "version": "==8.3.2" + "version": "==8.3.3" }, "oslo.i18n": { "hashes": [ @@ -348,17 +346,17 @@ }, "oslo.utils": { "hashes": [ - "sha256:12f5b6fecd77422ad83e5822a8289d96b35a3c5a0c9e8892687badefb1516891", - "sha256:460065e6d8bff195df2b543ed0f27589be3bef233b6df98f1b9157d200102fd7" + "sha256:51c1127c518e44beb4b565eb86f6f33e0a441d9a3e98eaff9f58a20933687b5d", + "sha256:b95b02c60dd4fac00d46525f73d27dc4e196c19281215aea84e8d389466b1b90" ], - "version": "==4.6.0" + "version": "==4.7.0" }, "packaging": { "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236", + "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376" ], - "version": "==20.4" + "version": "==20.7" }, "pbr": { "hashes": [ @@ -391,10 +389,10 @@ }, "pyparsing": { "hashes": [ - "sha256:1060635ca5ac864c2b7bc7b05a448df4e32d7d8c65e33cbe1514810d339672a2", - "sha256:56a551039101858c9e189ac9e66e330a03fb7079e97ba6b50193643905f450ce" + "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", + "sha256:38891c1032d0c759f0fa5ed3a8f249fd992b083fa2303ead58ee48a51b269e02" ], - "version": "==3.0.0a2" + "version": "==3.0.0b1" }, "python-dateutil": { "hashes": [ @@ -412,21 +410,23 @@ }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268", + "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd" ], - "version": "==2020.1" + "version": "==2020.4" }, "pyyaml": { "hashes": [ "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e", "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a", "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" @@ -435,10 +435,10 @@ }, "requests": { "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8", + "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998" ], - "version": "==2.24.0" + "version": "==2.25.0" }, "rfc3986": { "hashes": [ @@ -463,18 +463,18 @@ }, "stevedore": { "hashes": [ - "sha256:5e1ab03eaae06ef6ce23859402de785f08d97780ed774948ef16c4652c41bc62", - "sha256:f845868b3a3a77a2489d226568abe7328b5c2d4f6a011cc759dfa99144a521f0" + "sha256:3a5bbd0652bf552748871eaa73a4a8dc2899786bc497a2aa1fcb4dcdb0debeee", + "sha256:50d7b78fbaf0d04cd62411188fa7eedcb03eb7f4c4b37005615ceebe582aa82a" ], - "version": "==3.2.2" + "version": "==3.3.0" }, "urllib3": { "hashes": [ - "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", - "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" ], "markers": "python_version != '3.4'", - "version": "==1.25.11" + "version": "==1.26.2" }, "werkzeug": { "hashes": [ @@ -498,14 +498,21 @@ }, "zipp": { "hashes": [ - "sha256:16522f69653f0d67be90e8baa4a46d66389145b734345d68a257da53df670903", - "sha256:c1532a8030c32fd52ff6a288d855fe7adef5823ba1d26a29a68fd6314aa72baa" + "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", + "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" ], "markers": "python_version < '3.8'", - "version": "==3.3.1" + "version": "==3.4.0" } }, "develop": { + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "version": "==1.4.0" + }, "atomicwrites": { "hashes": [ "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", @@ -515,17 +522,17 @@ }, "attrs": { "hashes": [ - "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", - "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" + "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", + "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" ], - "version": "==20.2.0" + "version": "==20.3.0" }, "certifi": { "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" + "sha256:1f422849db327d534e3d0c5f02a263458c3955ec0aae4ff09b95f195c59f4edd", + "sha256:f05def092c44fbf25834a51509ef6e631dc19765ab8a57b4e7ab85531f0a9cf4" ], - "version": "==2020.6.20" + "version": "==2020.11.8" }, "chardet": { "hashes": [ @@ -536,10 +543,10 @@ }, "click": { "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + "sha256:681c9380a24b22fec089c8e5ffe40aa16a0da79f248a26fe2481bfa8170bfcc1", + "sha256:e4315a188403c0258bbc4a4e31863e48fc301c4e95b8007a8eeda0391158df13" ], - "version": "==7.1.2" + "version": "==8.0.0a1" }, "codacy-coverage": { "hashes": [ @@ -566,11 +573,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da", - "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3" + "sha256:590690d61efdd716ff82c39ca9a9d4209252adfe288a4b5721181050acbd4175", + "sha256:d9b8a46a0885337627a6430db287176970fff18ad421becec1d64cfc763c2099" ], "markers": "python_version < '3.8'", - "version": "==2.0.0" + "version": "==3.1.0" }, "itsdangerous": { "hashes": [ @@ -586,6 +593,13 @@ ], "version": "==3.0.0a1" }, + "linecache2": { + "hashes": [ + "sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c", + "sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef" + ], + "version": "==1.0.0" + }, "markupsafe": { "hashes": [ "sha256:06358015a4dee8ee23ae426bf885616ab3963622defd829eb45b44e3dee3515f", @@ -615,18 +629,18 @@ }, "more-itertools": { "hashes": [ - "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", - "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" + "sha256:8e1a2a43b2f2727425f2b5839587ae37093f19153dc26c0927d1048ff6557330", + "sha256:b3a9005928e5bed54076e6e549c792b306fddfe72b2d1d22dd63d42d5d3899cf" ], "markers": "python_version > '2.7'", - "version": "==8.5.0" + "version": "==8.6.0" }, "packaging": { "hashes": [ - "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", - "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" + "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236", + "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376" ], - "version": "==20.4" + "version": "==20.7" }, "pluggy": { "hashes": [ @@ -644,10 +658,10 @@ }, "pyparsing": { "hashes": [ - "sha256:1060635ca5ac864c2b7bc7b05a448df4e32d7d8c65e33cbe1514810d339672a2", - "sha256:56a551039101858c9e189ac9e66e330a03fb7079e97ba6b50193643905f450ce" + "sha256:13140e8d0e1edd806eb50f18535d77f2143b40771d4aaef6b4950dd93d48a7db", + "sha256:38891c1032d0c759f0fa5ed3a8f249fd992b083fa2303ead58ee48a51b269e02" ], - "version": "==3.0.0a2" + "version": "==3.0.0b1" }, "pytest": { "hashes": [ @@ -675,10 +689,10 @@ }, "requests": { "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" + "sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8", + "sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998" ], - "version": "==2.24.0" + "version": "==2.25.0" }, "six": { "hashes": [ @@ -687,13 +701,28 @@ ], "version": "==1.15.0" }, + "traceback2": { + "hashes": [ + "sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030", + "sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23" + ], + "version": "==1.4.0" + }, + "unittest2": { + "hashes": [ + "sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8", + "sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579" + ], + "index": "pypi", + "version": "==1.1.0" + }, "urllib3": { "hashes": [ - "sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2", - "sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e" + "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", + "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" ], "markers": "python_version != '3.4'", - "version": "==1.25.11" + "version": "==1.26.2" }, "wcwidth": { "hashes": [ @@ -711,11 +740,11 @@ }, "zipp": { "hashes": [ - "sha256:16522f69653f0d67be90e8baa4a46d66389145b734345d68a257da53df670903", - "sha256:c1532a8030c32fd52ff6a288d855fe7adef5823ba1d26a29a68fd6314aa72baa" + "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108", + "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb" ], "markers": "python_version < '3.8'", - "version": "==3.3.1" + "version": "==3.4.0" } } } diff --git a/manifestservice/manifests/__init__.py b/manifestservice/manifests/__init__.py index bb2f1e3..f26fd8b 100644 --- a/manifestservice/manifests/__init__.py +++ b/manifestservice/manifests/__init__.py @@ -13,6 +13,7 @@ blueprint = flask.Blueprint("manifests", __name__) + @blueprint.route("/", methods=["GET"]) def get_manifests(): """ @@ -123,11 +124,12 @@ def put_manifest(): return flask.jsonify(ret), 200 + @blueprint.route("/cohorts", methods=["GET"]) def get_cohorts(): """ - Returns a list of filenames -- which are GUIDs -- corresponding to the user's exported - PFBs. We find the appropriate folder ("prefix") in the bucket by asking Fence for + Returns a list of filenames -- which are GUIDs -- corresponding to the user's exported + PFBs. We find the appropriate folder ("prefix") in the bucket by asking Fence for info about the user's access token. --- responses: @@ -154,6 +156,7 @@ def get_cohorts(): return flask.jsonify(json_to_return), 200 + @blueprint.route("/cohorts", methods=["PUT", "POST"]) def put_pfb_guid(): """ @@ -346,11 +349,11 @@ def _list_files_in_bucket(bucket_name, folder): Lists the files in an s3 bucket. Returns a dictionary. The return value is of the form { - "manifests:" [ + "manifests:" [ # For files in the root of the user folder { "filename": , "last_modified": }, ... ], - "cohorts": [ + "cohorts": [ # For files in the cohorts/ folder { "filename": , "last_modified": }, ... ] @@ -371,15 +374,18 @@ def _list_files_in_bucket(bucket_name, folder): bucket_objects = bucket.objects.filter(Prefix=folder + "/") for object_summary in bucket_objects: file_marker = { - "filename": ntpath.basename(object_summary.key), "last_modified": object_summary.last_modified.strftime( "%Y-%m-%d %H:%M:%S" ), - "last_modified_timestamp": datetime.timestamp(object_summary.last_modified), + "last_modified_timestamp": datetime.timestamp( + object_summary.last_modified + ), } if not "cohorts/" in object_summary.key: + file_marker["filename"] = ntpath.basename(object_summary.key) manifests.append(file_marker) else: + file_marker["filename"] = object_summary.key.split("cohorts/")[1] guids.append(file_marker) except Exception as e: logger.error( @@ -388,9 +394,9 @@ def _list_files_in_bucket(bucket_name, folder): ) ) return str(e), False - - manifests_sorted = sorted(manifests, key = lambda i: i['last_modified_timestamp']) - guids_sorted = sorted(guids, key = lambda i: i['last_modified_timestamp']) + + manifests_sorted = sorted(manifests, key=lambda i: i["last_modified_timestamp"]) + guids_sorted = sorted(guids, key=lambda i: i["last_modified_timestamp"]) rv = {"manifests": manifests_sorted, "cohorts": guids_sorted} return rv, True diff --git a/tests/broken_s3_test.py b/tests/broken_s3_test.py deleted file mode 100644 index d02cb6e..0000000 --- a/tests/broken_s3_test.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest -import requests -import json as json_utils -import random -from manifestservice import manifests -import boto3 - -from manifestservice.api import create_app - -mocks = {} - - -@pytest.fixture -def app(mocker): - test_user = { - "context": { - "user": { - "policies": [ - "data_upload", - "programs.test-read-storage", - "programs.test-read", - ], - "google": {"proxy_group": None}, - "is_admin": True, - "name": "example@uchicago.edu", - "projects": { - "test": [ - "read-storage", - "read", - "create", - "write-storage", - "upload", - "update", - "delete", - ] - }, - } - }, - "aud": ["data", "user", "fence", "openid"], - "sub": "18", - } - - mocks["current_token"] = mocker.patch( - "manifestservice.manifests.current_token", return_value=test_user - ) - - mocks["_authenticate_user"] = mocker.patch( - "manifestservice.manifests._authenticate_user", return_value=(None, 200) - ) - - broken_s3_connection = boto3.Session('a', 'b', 'c') - - mocks["boto3"] = mocker.patch( - "manifestservice.manifests.boto3.Session", - return_value=broken_s3_connection - ) - - app = create_app() - return app - - -def test_GET_cohorts(client): - """ - Test GET /cohorts if s3 creds are broken - """ - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - r = client.get("/cohorts", headers=headers) - - assert r.status_code == 500 - assert mocks["_authenticate_user"].call_count == 1 - - response = r.json - assert len(response.keys()) == 1 - assert response["error"] == "Currently unable to connect to s3." \ No newline at end of file diff --git a/tests/cohorts_test.py b/tests/cohorts_test.py new file mode 100644 index 0000000..b5fd2b5 --- /dev/null +++ b/tests/cohorts_test.py @@ -0,0 +1,85 @@ +import json as json_utils + +from manifestservice.manifests import _list_files_in_bucket + + +def test_POST_successful_GUID_add(client, mocks): + """ + Test the Export PFB to Workspace pathway: a cohort is added to the bucket. + Note that because s3 is being mocked, only an integration test can properly + verify file creation. + """ + test_guid = "5183a350-9d56-4084-8a03-6471cafeb7fe" + post_body = {"guid": test_guid} + + headers = {"Content-Type": "application/json", "Accept": "application/json"} + r = client.post("/cohorts", data=json_utils.dumps(post_body), headers=headers) + + assert r.status_code == 200 + assert mocks["_authenticate_user"].call_count == 1 + assert mocks["_get_file_contents"].call_count == 0 + assert mocks["_add_manifest_to_bucket"].call_count == 0 + assert mocks["_add_GUID_to_bucket"].call_count == 1 + + json = r.json + new_guid = json["filename"] + + assert new_guid is not None + assert type(new_guid) is str + + +def test_GET_cohorts(client, mocks): + """ + Test GET /cohorts + """ + headers = {"Content-Type": "application/json", "Accept": "application/json"} + r = client.get("/cohorts", headers=headers) + + assert r.status_code == 200 + assert mocks["_authenticate_user"].call_count == 1 + assert mocks["_get_file_contents"].call_count == 0 + assert mocks["_add_manifest_to_bucket"].call_count == 0 + assert mocks["_add_GUID_to_bucket"].call_count == 0 + assert mocks["_list_files_in_bucket"].call_count == 1 + + json = r.json + cohorts_returned = json["cohorts"] + assert len(cohorts_returned) == 1 + # From the s3 mock + assert cohorts_returned[0]["filename"] == "18e32c12-a053-4ac5-90a5-f01f70b5c2be" + + +def test_GET_cohorts_broken_s3(client, broken_s3_mocks): + """ + Test GET /cohorts if s3 creds are broken + """ + headers = {"Content-Type": "application/json", "Accept": "application/json"} + r = client.get("/cohorts", headers=headers) + + assert r.status_code == 500 + assert broken_s3_mocks["_authenticate_user"].call_count == 1 + + response = r.json + assert len(response.keys()) == 1 + assert response["error"] == "Currently unable to connect to s3." + + +def test_list_files_in_bucket(client, mocked_bucket): + """ + Test that prefixes are not removed from GUIDs when listing cohorts + in buckets + """ + result, ok = _list_files_in_bucket("fake_bucket_name", "fake_folder") + assert ok, result + + manifests = result["manifests"] + assert len(manifests) == 1 + assert manifests[0]["filename"] == "my-manifest.json" + + cohorts = result["cohorts"] + assert len(cohorts) == 2 + for cohort in cohorts: + if "without-prefix" in cohort["filename"]: + assert cohort["filename"] == "guid-without-prefix" + else: + assert cohort["filename"] == "dg.mytest/guid-with-prefix" diff --git a/tests/conftest.py b/tests/conftest.py index efd8b2b..ce9be61 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,153 @@ +import boto3 +from datetime import datetime import pytest +from unittest.mock import MagicMock, patch -from manifestservice.api import app as service_app +from manifestservice.api import create_app -@pytest.fixture(scope="session") +@pytest.fixture def app(): - # load configuration - # service_app.config.from_object('manifestservice.test_settings') - # app_init(service_app) - return service_app + app = create_app() + return app + + +@pytest.fixture +def mocks(mocker): + test_user = { + "context": { + "user": { + "policies": [ + "data_upload", + "programs.test-read-storage", + "programs.test-read", + ], + "google": {"proxy_group": None}, + "is_admin": True, + "name": "example@uchicago.edu", + "projects": { + "test": [ + "read-storage", + "read", + "create", + "write-storage", + "upload", + "update", + "delete", + ] + }, + } + }, + "aud": ["data", "user", "fence", "openid"], + "sub": "18", + } + all_mocks = {} + + all_mocks["current_token"] = mocker.patch( + "manifestservice.manifests.current_token", return_value=test_user + ) + + all_mocks["_authenticate_user"] = mocker.patch( + "manifestservice.manifests._authenticate_user", return_value=(None, 200) + ) + + all_mocks["_list_files_in_bucket"] = mocker.patch( + "manifestservice.manifests._list_files_in_bucket", + return_value=( + { + "manifests": [ + {"filename": "manifest-a-b-c.json"}, + ], + "cohorts": [{"filename": "18e32c12-a053-4ac5-90a5-f01f70b5c2be"}], + }, + True, + ), + ) + + all_mocks["_add_manifest_to_bucket"] = mocker.patch( + "manifestservice.manifests._add_manifest_to_bucket", + return_value=("manifest-xxx.json", True), + ) + + all_mocks["_get_file_contents"] = mocker.patch( + "manifestservice.manifests._get_file_contents", return_value="" + ) + + all_mocks["_add_GUID_to_bucket"] = mocker.patch( + "manifestservice.manifests._add_GUID_to_bucket", + return_value=("a-guid-value", True), + ) + + return all_mocks + + +@pytest.fixture +def broken_s3_mocks(mocker): + test_user = { + "context": { + "user": { + "policies": [ + "data_upload", + "programs.test-read-storage", + "programs.test-read", + ], + "google": {"proxy_group": None}, + "is_admin": True, + "name": "example@uchicago.edu", + "projects": { + "test": [ + "read-storage", + "read", + "create", + "write-storage", + "upload", + "update", + "delete", + ] + }, + } + }, + "aud": ["data", "user", "fence", "openid"], + "sub": "18", + } + all_mocks = {} + + all_mocks["current_token"] = mocker.patch( + "manifestservice.manifests.current_token", return_value=test_user + ) + + all_mocks["_authenticate_user"] = mocker.patch( + "manifestservice.manifests._authenticate_user", return_value=(None, 200) + ) + + broken_s3_connection = boto3.Session("a", "b", "c") + + all_mocks["boto3"] = mocker.patch( + "manifestservice.manifests.boto3.Session", return_value=broken_s3_connection + ) + + return all_mocks + + +@pytest.fixture +def mocked_bucket(): + class MockedS3Object: + def __init__(self, key): + self.key = key + self.last_modified = datetime.now() + + mock = MagicMock() + mock.return_value = iter( + [ + MockedS3Object(key="username/my-manifest.json"), + MockedS3Object(key="username/cohorts/guid-without-prefix"), + MockedS3Object(key="username/cohorts/dg.mytest/guid-with-prefix"), + ] + ) + + patcher = patch("boto3.resources.collection.ResourceCollection.__iter__", mock) + patcher.start() + + yield mock + + patcher.stop() diff --git a/tests/app_test.py b/tests/manifests_test.py similarity index 58% rename from tests/app_test.py rename to tests/manifests_test.py index 1f50e26..3b8804c 100644 --- a/tests/app_test.py +++ b/tests/manifests_test.py @@ -1,81 +1,7 @@ -import pytest -import requests import json as json_utils import random from manifestservice import manifests -from manifestservice.api import create_app - -mocks = {} - - -@pytest.fixture -def app(mocker): - test_user = { - "context": { - "user": { - "policies": [ - "data_upload", - "programs.test-read-storage", - "programs.test-read", - ], - "google": {"proxy_group": None}, - "is_admin": True, - "name": "example@uchicago.edu", - "projects": { - "test": [ - "read-storage", - "read", - "create", - "write-storage", - "upload", - "update", - "delete", - ] - }, - } - }, - "aud": ["data", "user", "fence", "openid"], - "sub": "18", - } - - mocks["current_token"] = mocker.patch( - "manifestservice.manifests.current_token", return_value=test_user - ) - - mocks["_authenticate_user"] = mocker.patch( - "manifestservice.manifests._authenticate_user", return_value=(None, 200) - ) - - mocks["_list_files_in_bucket"] = mocker.patch( - "manifestservice.manifests._list_files_in_bucket", - return_value=({ - "manifests": [ - {"filename": "manifest-a-b-c.json"}, - ], - "cohorts": [ - {"filename": "18e32c12-a053-4ac5-90a5-f01f70b5c2be"} - ] - }, True), - ) - - mocks["_add_manifest_to_bucket"] = mocker.patch( - "manifestservice.manifests._add_manifest_to_bucket", - return_value=("manifest-xxx.json", True), - ) - - mocks["_get_file_contents"] = mocker.patch( - "manifestservice.manifests._get_file_contents", return_value="" - ) - - mocks["_add_GUID_to_bucket"] = mocker.patch( - "manifestservice.manifests._add_GUID_to_bucket", - return_value=("a-guid-value", True), - ) - - app = create_app() - return app - def test_generate_unique_manifest_filename_basic_date_generation(): """ @@ -153,7 +79,7 @@ def test_is_valid_manifest(): assert is_valid is True -def test_POST_handles_invalid_json(client): +def test_POST_handles_invalid_json(client, mocks): """ Test that we get a 400 if flask.request.json is not filled in. """ @@ -161,7 +87,7 @@ def test_POST_handles_invalid_json(client): assert r.status_code == 400 -def test_POST_handles_invalid_manifest_keys(client): +def test_POST_handles_invalid_manifest_keys(client, mocks): """ Test that we get a 400 if the manifest is missing the required key -- object_id. """ @@ -175,14 +101,14 @@ def test_POST_handles_invalid_manifest_keys(client): assert r.status_code == 400 -def test_POST_successful_manifest_upload(client): +def test_POST_successful_manifest_upload(client, mocks): """ Test the full user pathway: a manifest is created, listed, and then downloaded. Unfortunately, we cannot verify here that the manifest is present in the listed files, nor that the filebody is correct, as that would require a real s3 connection. Instead, s3 is mocked and we assert that the correct functions are called. """ - + random_nums = [ random.randint(1, 101), random.randint(1, 101), @@ -225,49 +151,3 @@ def test_POST_successful_manifest_upload(client): assert mocks["_add_manifest_to_bucket"].call_count == 1 assert mocks["_list_files_in_bucket"].call_count == 1 assert mocks["_get_file_contents"].call_count == 1 - -def test_POST_successful_GUID_add(client): - """ - Test the Export PFB to Workspace pathway: a cohort is added to the bucket. - Note that because s3 is being mocked, only an integration test can properly - verify file creation. - """ - - test_guid = "5183a350-9d56-4084-8a03-6471cafeb7fe" - post_body = { "guid" : test_guid } - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - r = client.post("/cohorts", data=json_utils.dumps(post_body), headers=headers) - - assert r.status_code == 200 - assert mocks["_authenticate_user"].call_count == 1 - assert mocks["_get_file_contents"].call_count == 0 - assert mocks["_add_manifest_to_bucket"].call_count == 0 - assert mocks["_add_GUID_to_bucket"].call_count == 1 - - json = r.json - new_guid = json["filename"] - - assert new_guid is not None - assert type(new_guid) is str - -def test_GET_cohorts(client): - """ - Test GET /cohorts - """ - - headers = {"Content-Type": "application/json", "Accept": "application/json"} - r = client.get("/cohorts", headers=headers) - - assert r.status_code == 200 - assert mocks["_authenticate_user"].call_count == 1 - assert mocks["_get_file_contents"].call_count == 0 - assert mocks["_add_manifest_to_bucket"].call_count == 0 - assert mocks["_add_GUID_to_bucket"].call_count == 0 - assert mocks["_list_files_in_bucket"].call_count == 1 - - json = r.json - cohorts_returned = json["cohorts"] - assert len(cohorts_returned) == 1 - # From the s3 mock - assert cohorts_returned[0]["filename"] == "18e32c12-a053-4ac5-90a5-f01f70b5c2be" \ No newline at end of file