diff --git a/configs/test/project.yaml b/configs/test/project.yaml index 378b6dbd97..7b362ae339 100644 --- a/configs/test/project.yaml +++ b/configs/test/project.yaml @@ -75,6 +75,9 @@ env: # Application ID for the Google Cloud Project. In production, this will have a s~ prefix. APPLICATION_ID: test-clusterfuzz + # GCP project where the GKE cluster running cronjobs will be. + K8S_PROJECT: test-clusterfuzz + # Default project name unless overridden in a job definition. PROJECT_NAME: test-project diff --git a/configs/test/pubsub/queues.yaml b/configs/test/pubsub/queues.yaml index e300ea5c80..a49c8e8652 100644 --- a/configs/test/pubsub/queues.yaml +++ b/configs/test/pubsub/queues.yaml @@ -66,3 +66,9 @@ resources: type: queue.jinja - name: ml-jobs-linux type: queue.jinja + - name: utask_main + type: queue.jinja + - name: preprocess + type: queue.jinja + - name: postprocess + type: queue.jinja diff --git a/src/Pipfile b/src/Pipfile index 31f36bb149..0716afa5dc 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -36,7 +36,7 @@ pyOpenSSL = "==22.0.0" python-dateutil = "==2.8.1" PyYAML = "==6.0" pytz = "==2023.3" -redis = "==3.3.11" +redis = "==4.6.0" requests = "==2.21.0" sendgrid = "==6.0.4" wrapt = "==1.16.0" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 08bcf57ad5..a08b14252b 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "56cf16e80c505055e8abe7d818fff520f19feb1682590fd782580d847eee9006" + "sha256": "34863be9ebbc59b6df81493d51760b6aaf0648d8029638fd65cf0915139cb0dd" }, "pipfile-spec": 6, "requires": {}, @@ -16,11 +16,11 @@ "default": { "aiohappyeyeballs": { "hashes": [ - "sha256:4ca893e6c5c1f5bf3888b04cb5a3bee24995398efef6e0b9f747b5e89d84fd74", - "sha256:8522691d9a154ba1145b157d6d5c15e5c692527ce6a53c5e5f9876977f6dab2f" + "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", + "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8" ], - "markers": "python_version >= '3.8'", - "version": "==2.4.2" + "markers": "python_version >= '3.9'", + "version": "==2.6.1" }, "aiohttp": { "hashes": [ @@ -121,11 +121,11 @@ }, "aiosignal": { "hashes": [ - "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc", - "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17" + "sha256:45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5", + "sha256:a8c255c66fafb1e499c9351d0bf32ff2d8a0321595ebac3b93713656d2436f54" ], - "markers": "python_version >= '3.7'", - "version": "==1.3.1" + "markers": "python_version >= '3.9'", + "version": "==1.3.2" }, "antlr4-python3-runtime": { "hashes": [ @@ -136,11 +136,11 @@ }, "attrs": { "hashes": [ - "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", + "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b" ], - "markers": "python_version >= '3.7'", - "version": "==24.2.0" + "markers": "python_version >= '3.8'", + "version": "==25.3.0" }, "bcrypt": { "hashes": [ @@ -177,11 +177,11 @@ }, "cachetools": { "hashes": [ - "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292", - "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a" + "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", + "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" ], "markers": "python_version >= '3.7'", - "version": "==5.5.0" + "version": "==5.5.2" }, "certifi": { "hashes": [ @@ -306,11 +306,11 @@ }, "deprecated": { "hashes": [ - "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c", - "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3" + "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", + "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.2.14" + "version": "==1.2.18" }, "distro": { "hashes": [ @@ -322,86 +322,101 @@ }, "frozenlist": { "hashes": [ - "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7", - "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98", - "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad", - "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5", - "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae", - "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e", - "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a", - "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701", - "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d", - "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6", - "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6", - "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106", - "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75", - "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868", - "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a", - "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0", - "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1", - "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826", - "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec", - "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6", - "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950", - "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19", - "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0", - "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8", - "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a", - "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09", - "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86", - "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c", - "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5", - "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b", - "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b", - "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d", - "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0", - "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea", - "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776", - "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a", - "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897", - "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7", - "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09", - "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9", - "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe", - "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd", - "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742", - "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09", - "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0", - "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932", - "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1", - "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a", - "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49", - "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d", - "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7", - "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480", - "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89", - "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e", - "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b", - "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82", - "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb", - "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068", - "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8", - "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b", - "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb", - "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2", - "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11", - "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b", - "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc", - "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0", - "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497", - "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17", - "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0", - "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2", - "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439", - "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5", - "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac", - "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825", - "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887", - "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced", - "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74" + "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e", + "sha256:03d33c2ddbc1816237a67f66336616416e2bbb6beb306e5f890f2eb22b959cdf", + "sha256:04a5c6babd5e8fb7d3c871dc8b321166b80e41b637c31a995ed844a6139942b6", + "sha256:0996c66760924da6e88922756d99b47512a71cfd45215f3570bf1e0b694c206a", + "sha256:0cc974cc93d32c42e7b0f6cf242a6bd941c57c61b618e78b6c0a96cb72788c1d", + "sha256:0f253985bb515ecd89629db13cb58d702035ecd8cfbca7d7a7e29a0e6d39af5f", + "sha256:11aabdd62b8b9c4b84081a3c246506d1cddd2dd93ff0ad53ede5defec7886b28", + "sha256:12f78f98c2f1c2429d42e6a485f433722b0061d5c0b0139efa64f396efb5886b", + "sha256:140228863501b44b809fb39ec56b5d4071f4d0aa6d216c19cbb08b8c5a7eadb9", + "sha256:1431d60b36d15cda188ea222033eec8e0eab488f39a272461f2e6d9e1a8e63c2", + "sha256:15538c0cbf0e4fa11d1e3a71f823524b0c46299aed6e10ebb4c2089abd8c3bec", + "sha256:15b731db116ab3aedec558573c1a5eec78822b32292fe4f2f0345b7f697745c2", + "sha256:17dcc32fc7bda7ce5875435003220a457bcfa34ab7924a49a1c19f55b6ee185c", + "sha256:1893f948bf6681733aaccf36c5232c231e3b5166d607c5fa77773611df6dc336", + "sha256:189f03b53e64144f90990d29a27ec4f7997d91ed3d01b51fa39d2dbe77540fd4", + "sha256:1a8ea951bbb6cacd492e3948b8da8c502a3f814f5d20935aae74b5df2b19cf3d", + "sha256:1b96af8c582b94d381a1c1f51ffaedeb77c821c690ea5f01da3d70a487dd0a9b", + "sha256:1e76bfbc72353269c44e0bc2cfe171900fbf7f722ad74c9a7b638052afe6a00c", + "sha256:2150cc6305a2c2ab33299453e2968611dacb970d2283a14955923062c8d00b10", + "sha256:226d72559fa19babe2ccd920273e767c96a49b9d3d38badd7c91a0fdeda8ea08", + "sha256:237f6b23ee0f44066219dae14c70ae38a63f0440ce6750f868ee08775073f942", + "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8", + "sha256:2b5e23253bb709ef57a8e95e6ae48daa9ac5f265637529e4ce6b003a37b2621f", + "sha256:2d0da8bbec082bf6bf18345b180958775363588678f64998c2b7609e34719b10", + "sha256:2f3f7a0fbc219fb4455264cae4d9f01ad41ae6ee8524500f381de64ffaa077d5", + "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6", + "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21", + "sha256:31a9ac2b38ab9b5a8933b693db4939764ad3f299fcaa931a3e605bc3460e693c", + "sha256:366d8f93e3edfe5a918c874702f78faac300209a4d5bf38352b2c1bdc07a766d", + "sha256:374ca2dabdccad8e2a76d40b1d037f5bd16824933bf7bcea3e59c891fd4a0923", + "sha256:44c49271a937625619e862baacbd037a7ef86dd1ee215afc298a417ff3270608", + "sha256:45e0896250900b5aa25180f9aec243e84e92ac84bd4a74d9ad4138ef3f5c97de", + "sha256:498524025a5b8ba81695761d78c8dd7382ac0b052f34e66939c42df860b8ff17", + "sha256:50cf5e7ee9b98f22bdecbabf3800ae78ddcc26e4a435515fc72d97903e8488e0", + "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f", + "sha256:561eb1c9579d495fddb6da8959fd2a1fca2c6d060d4113f5844b433fc02f2641", + "sha256:5a3ba5f9a0dfed20337d3e966dc359784c9f96503674c2faf015f7fe8e96798c", + "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a", + "sha256:5c28f4b5dbef8a0d8aad0d4de24d1e9e981728628afaf4ea0792f5d0939372f0", + "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9", + "sha256:6321899477db90bdeb9299ac3627a6a53c7399c8cd58d25da094007402b039ab", + "sha256:6482a5851f5d72767fbd0e507e80737f9c8646ae7fd303def99bfe813f76cf7f", + "sha256:666534d15ba8f0fda3f53969117383d5dc021266b3c1a42c9ec4855e4b58b9d3", + "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a", + "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784", + "sha256:73f2e31ea8dd7df61a359b731716018c2be196e5bb3b74ddba107f694fbd7604", + "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d", + "sha256:76e4753701248476e6286f2ef492af900ea67d9706a0155335a40ea21bf3b2f5", + "sha256:7707a25d6a77f5d27ea7dc7d1fc608aa0a478193823f88511ef5e6b8a48f9d03", + "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e", + "sha256:7a1a048f9215c90973402e26c01d1cff8a209e1f1b53f72b95c13db61b00f953", + "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee", + "sha256:7f3c8c1dacd037df16e85227bac13cca58c30da836c6f936ba1df0c05d046d8d", + "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817", + "sha256:828afae9f17e6de596825cf4228ff28fbdf6065974e5ac1410cecc22f699d2b3", + "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039", + "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f", + "sha256:90646abbc7a5d5c7c19461d2e3eeb76eb0b204919e6ece342feb6032c9325ae9", + "sha256:91d6c171862df0a6c61479d9724f22efb6109111017c87567cfeb7b5d1449fdf", + "sha256:9272fa73ca71266702c4c3e2d4a28553ea03418e591e377a03b8e3659d94fa76", + "sha256:92b5278ed9d50fe610185ecd23c55d8b307d75ca18e94c0e7de328089ac5dcba", + "sha256:97160e245ea33d8609cd2b8fd997c850b56db147a304a262abc2b3be021a9171", + "sha256:977701c081c0241d0955c9586ffdd9ce44f7a7795df39b9151cd9a6fd0ce4cfb", + "sha256:9b7dc0c4338e6b8b091e8faf0db3168a37101943e687f373dce00959583f7439", + "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631", + "sha256:9bbcdfaf4af7ce002694a4e10a0159d5a8d20056a12b05b45cea944a4953f972", + "sha256:9c2623347b933fcb9095841f1cc5d4ff0b278addd743e0e966cb3d460278840d", + "sha256:a2fe128eb4edeabe11896cb6af88fca5346059f6c8d807e3b910069f39157869", + "sha256:a72b7a6e3cd2725eff67cd64c8f13335ee18fc3c7befc05aed043d24c7b9ccb9", + "sha256:a9fe0f1c29ba24ba6ff6abf688cb0b7cf1efab6b6aa6adc55441773c252f7411", + "sha256:b97f7b575ab4a8af9b7bc1d2ef7f29d3afee2226bd03ca3875c16451ad5a7723", + "sha256:bdac3c7d9b705d253b2ce370fde941836a5f8b3c5c2b8fd70940a3ea3af7f4f2", + "sha256:c03eff4a41bd4e38415cbed054bbaff4a075b093e2394b6915dca34a40d1e38b", + "sha256:c16d2fa63e0800723139137d667e1056bee1a1cf7965153d2d104b62855e9b99", + "sha256:c1fac3e2ace2eb1052e9f7c7db480818371134410e1f5c55d65e8f3ac6d1407e", + "sha256:ce3aa154c452d2467487765e3adc730a8c153af77ad84096bc19ce19a2400840", + "sha256:cee6798eaf8b1416ef6909b06f7dc04b60755206bddc599f52232606e18179d3", + "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb", + "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3", + "sha256:dd47a5181ce5fcb463b5d9e17ecfdb02b678cca31280639255ce9d0e5aa67af0", + "sha256:dd94994fc91a6177bfaafd7d9fd951bc8689b0a98168aa26b5f543868548d3ca", + "sha256:de537c11e4aa01d37db0d403b57bd6f0546e71a82347a97c6a9f0dcc532b3a45", + "sha256:df6e2f325bfee1f49f81aaac97d2aa757c7646534a06f8f577ce184afe2f0a9e", + "sha256:e66cc454f97053b79c2ab09c17fbe3c825ea6b4de20baf1be28919460dd7877f", + "sha256:e79225373c317ff1e35f210dd5f1344ff31066ba8067c307ab60254cd3a78ad5", + "sha256:f1577515d35ed5649d52ab4319db757bb881ce3b2b796d7283e6634d99ace307", + "sha256:f1e6540b7fa044eee0bb5111ada694cf3dc15f2b0347ca125ee9ca984d5e9e6e", + "sha256:f2ac49a9bedb996086057b75bf93538240538c6d9b38e57c82d51f75a73409d2", + "sha256:f47c9c9028f55a04ac254346e92977bf0f166c483c74b4232bee19a6697e4778", + "sha256:f5f9da7f5dbc00a604fe74aa02ae7c98bcede8a3b8b9666f9f86fc13993bc71a", + "sha256:fd74520371c3c4175142d02a976aee0b4cb4a7cc912a60586ffd8d5929979b30", + "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a" ], "markers": "python_version >= '3.8'", - "version": "==1.4.1" + "version": "==1.5.0" }, "future": { "hashes": [ @@ -451,19 +466,19 @@ }, "google-cloud-appengine-logging": { "hashes": [ - "sha256:344e0244404049b42164e4d6dc718ca2c81b393d066956e7cb85fd9407ed9c48", - "sha256:de7d766e5d67b19fc5833974b505b32d2a5bbdfb283fd941e320e7cfdae4cb83" + "sha256:48f4dcf43000899c7b411bc27181f70240e81a958a44e44ce800ba8e5d5184ac", + "sha256:f97bde36c7f7ff541123c2570813158bdda0c3f2385c8d32fdf1211c561ae56d" ], "markers": "python_version >= '3.7'", - "version": "==1.4.5" + "version": "==1.6.1" }, "google-cloud-audit-log": { "hashes": [ - "sha256:8340793120a1d5aa143605def8704ecdcead15106f754ef1381ae3bab533722f", - "sha256:901428b257020d8c1d1133e0fa004164a555e5a395c7ca3cdbb8486513df3a65" + "sha256:2598f1533a7d7cdd6c7bf448c12e5519c1d53162d78784e10bcdd1df67791bc3", + "sha256:daaedfb947a0d77f524e1bd2b560242ab4836fe1afd6b06b92f152b9658554ed" ], "markers": "python_version >= '3.7'", - "version": "==0.3.0" + "version": "==0.3.2" }, "google-cloud-batch": { "hashes": [ @@ -620,19 +635,19 @@ }, "googleapis-common-protos": { "hashes": [ - "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63", - "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0" + "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212", + "sha256:3e1b904a27a33c821b4b749fd31d334c0c9c30e6113023d495e48979a3dc9c5f" ], "markers": "python_version >= '3.7'", - "version": "==1.65.0" + "version": "==1.69.2" }, "grpc-google-iam-v1": { "hashes": [ - "sha256:3ff4b2fd9d990965e410965253c0da6f66205d5a8291c4c31c6ebecca18a9001", - "sha256:c3e86151a981811f30d5e7330f271cee53e73bb87755e88cc3b6f0c7b5fe374e" + "sha256:a3171468459770907926d56a440b2bb643eec1d7ba215f48f3ecece42b4d8351", + "sha256:b3e1fc387a1a329e41672197d0ace9de22c78dd7d215048c4c78712073f7bd20" ], "markers": "python_version >= '3.7'", - "version": "==0.13.1" + "version": "==0.14.2" }, "grpcio": { "hashes": [ @@ -748,101 +763,101 @@ }, "multidict": { "hashes": [ - "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f", - "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056", - "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761", - "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3", - "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b", - "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6", - "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748", - "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966", - "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f", - "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1", - "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6", - "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada", - "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305", - "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2", - "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d", - "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a", - "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef", - "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c", - "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb", - "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60", - "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6", - "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4", - "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478", - "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81", - "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7", - "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56", - "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3", - "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6", - "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30", - "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb", - "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506", - "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0", - "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925", - "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c", - "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6", - "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e", - "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95", - "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2", - "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133", - "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2", - "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa", - "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3", - "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3", - "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436", - "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657", - "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581", - "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492", - "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43", - "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2", - "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2", - "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926", - "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057", - "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc", - "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80", - "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255", - "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1", - "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972", - "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53", - "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1", - "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423", - "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a", - "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160", - "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c", - "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd", - "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa", - "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5", - "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b", - "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa", - "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef", - "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44", - "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4", - "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156", - "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753", - "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28", - "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d", - "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a", - "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304", - "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008", - "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429", - "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72", - "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399", - "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3", - "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392", - "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167", - "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c", - "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774", - "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351", - "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76", - "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875", - "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd", - "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28", - "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db" - ], - "markers": "python_version >= '3.8'", - "version": "==6.1.0" + "sha256:0085b0afb2446e57050140240a8595846ed64d1cbd26cef936bfab3192c673b8", + "sha256:042028348dc5a1f2be6c666437042a98a5d24cee50380f4c0902215e5ec41844", + "sha256:05fefbc3cddc4e36da209a5e49f1094bbece9a581faa7f3589201fd95df40e5d", + "sha256:063be88bd684782a0715641de853e1e58a2f25b76388538bd62d974777ce9bc2", + "sha256:07bfa8bc649783e703263f783f73e27fef8cd37baaad4389816cf6a133141331", + "sha256:08549895e6a799bd551cf276f6e59820aa084f0f90665c0f03dd3a50db5d3c48", + "sha256:095a2eabe8c43041d3e6c2cb8287a257b5f1801c2d6ebd1dd877424f1e89cf29", + "sha256:0b183a959fb88ad1be201de2c4bdf52fa8e46e6c185d76201286a97b6f5ee65c", + "sha256:0c383d28857f66f5aebe3e91d6cf498da73af75fbd51cedbe1adfb85e90c0460", + "sha256:0d57a01a2a9fa00234aace434d8c131f0ac6e0ac6ef131eda5962d7e79edfb5b", + "sha256:0dc25a3293c50744796e87048de5e68996104d86d940bb24bc3ec31df281b191", + "sha256:0e5a644e50ef9fb87878d4d57907f03a12410d2aa3b93b3acdf90a741df52c49", + "sha256:0f249badb360b0b4d694307ad40f811f83df4da8cef7b68e429e4eea939e49dd", + "sha256:0f74f2fc51555f4b037ef278efc29a870d327053aba5cb7d86ae572426c7cccc", + "sha256:125dd82b40f8c06d08d87b3510beaccb88afac94e9ed4a6f6c71362dc7dbb04b", + "sha256:13551d0e2d7201f0959725a6a769b6f7b9019a168ed96006479c9ac33fe4096b", + "sha256:14ed9ed1bfedd72a877807c71113deac292bf485159a29025dfdc524c326f3e1", + "sha256:163f4604e76639f728d127293d24c3e208b445b463168af3d031b92b0998bb90", + "sha256:19e2819b0b468174de25c0ceed766606a07cedeab132383f1e83b9a4e96ccb4f", + "sha256:1e2a2193d3aa5cbf5758f6d5680a52aa848e0cf611da324f71e5e48a9695cc86", + "sha256:1f3c099d3899b14e1ce52262eb82a5f5cb92157bb5106bf627b618c090a0eadc", + "sha256:214207dcc7a6221d9942f23797fe89144128a71c03632bf713d918db99bd36de", + "sha256:2325105e16d434749e1be8022f942876a936f9bece4ec41ae244e3d7fae42aaf", + "sha256:2529ddbdaa424b2c6c2eb668ea684dd6b75b839d0ad4b21aad60c168269478d7", + "sha256:256d431fe4583c5f1e0f2e9c4d9c22f3a04ae96009b8cfa096da3a8723db0a16", + "sha256:25bb96338512e2f46f615a2bb7c6012fe92a4a5ebd353e5020836a7e33120349", + "sha256:2e87f1926e91855ae61769ba3e3f7315120788c099677e0842e697b0bfb659f2", + "sha256:2fc6af8e39f7496047c7876314f4317736eac82bf85b54c7c76cf1a6f8e35d98", + "sha256:3157126b028c074951839233647bd0e30df77ef1fedd801b48bdcad242a60f4e", + "sha256:32c9b4878f48be3e75808ea7e499d6223b1eea6d54c487a66bc10a1871e3dc6a", + "sha256:32ed748ff9ac682eae7859790d3044b50e3076c7d80e17a44239683769ff485e", + "sha256:3501621d5e86f1a88521ea65d5cad0a0834c77b26f193747615b7c911e5422d2", + "sha256:437c33561edb6eb504b5a30203daf81d4a9b727e167e78b0854d9a4e18e8950b", + "sha256:48d39b1824b8d6ea7de878ef6226efbe0773f9c64333e1125e0efcfdd18a24c7", + "sha256:4ac3fcf9a2d369bd075b2c2965544036a27ccd277fc3c04f708338cc57533081", + "sha256:4ccfd74957ef53fa7380aaa1c961f523d582cd5e85a620880ffabd407f8202c0", + "sha256:52b05e21ff05729fbea9bc20b3a791c3c11da61649ff64cce8257c82a020466d", + "sha256:5389445f0173c197f4a3613713b5fb3f3879df1ded2a1a2e4bc4b5b9c5441b7e", + "sha256:5c5e7d2e300d5cb3b2693b6d60d3e8c8e7dd4ebe27cd17c9cb57020cac0acb80", + "sha256:5d26547423e5e71dcc562c4acdc134b900640a39abd9066d7326a7cc2324c530", + "sha256:5dd7106d064d05896ce28c97da3f46caa442fe5a43bc26dfb258e90853b39b44", + "sha256:5f8cb1329f42fadfb40d6211e5ff568d71ab49be36e759345f91c69d1033d633", + "sha256:61d5541f27533f803a941d3a3f8a3d10ed48c12cf918f557efcbf3cd04ef265c", + "sha256:639556758c36093b35e2e368ca485dada6afc2bd6a1b1207d85ea6dfc3deab27", + "sha256:641cf2e3447c9ecff2f7aa6e9eee9eaa286ea65d57b014543a4911ff2799d08a", + "sha256:6aed763b6a1b28c46c055692836879328f0b334a6d61572ee4113a5d0c859872", + "sha256:6e2a2d6749e1ff2c9c76a72c6530d5baa601205b14e441e6d98011000f47a7ac", + "sha256:7243c5a6523c5cfeca76e063efa5f6a656d1d74c8b1fc64b2cd1e84e507f7e2a", + "sha256:76b34c12b013d813e6cb325e6bd4f9c984db27758b16085926bbe7ceeaace626", + "sha256:781b5dd1db18c9e9eacc419027b0acb5073bdec9de1675c0be25ceb10e2ad133", + "sha256:7c611345bbe7cb44aabb877cb94b63e86f2d0db03e382667dbd037866d44b4f8", + "sha256:83b78c680d4b15d33042d330c2fa31813ca3974197bddb3836a5c635a5fd013f", + "sha256:84e87a7d75fa36839a3a432286d719975362d230c70ebfa0948549cc38bd5b46", + "sha256:89b3857652183b8206a891168af47bac10b970d275bba1f6ee46565a758c078d", + "sha256:8cd1a0644ccaf27e9d2f6d9c9474faabee21f0578fe85225cc5af9a61e1653df", + "sha256:8de4d42dffd5ced9117af2ce66ba8722402541a3aa98ffdf78dde92badb68932", + "sha256:94a7bb972178a8bfc4055db80c51efd24baefaced5e51c59b0d598a004e8305d", + "sha256:98aa8325c7f47183b45588af9c434533196e241be0a4e4ae2190b06d17675c02", + "sha256:9e658d1373c424457ddf6d55ec1db93c280b8579276bebd1f72f113072df8a5d", + "sha256:9f49585f4abadd2283034fc605961f40c638635bc60f5162276fec075f2e37a4", + "sha256:9f6cad071960ba1914fa231677d21b1b4a3acdcce463cee41ea30bc82e6040cf", + "sha256:a0cc398350ef31167e03f3ca7c19313d4e40a662adcb98a88755e4e861170bdd", + "sha256:a1133414b771619aa3c3000701c11b2e4624a7f492f12f256aedde97c28331a2", + "sha256:a33273a541f1e1a8219b2a4ed2de355848ecc0254264915b9290c8d2de1c74e1", + "sha256:a3c0ff89fe40a152e77b191b83282c9664357dce3004032d42e68c514ceff27e", + "sha256:a49994481b99cd7dedde07f2e7e93b1d86c01c0fca1c32aded18f10695ae17eb", + "sha256:abf5b17bc0cf626a8a497d89ac691308dbd825d2ac372aa990b1ca114e470151", + "sha256:ac380cacdd3b183338ba63a144a34e9044520a6fb30c58aa14077157a033c13e", + "sha256:ad81012b24b88aad4c70b2cbc2dad84018783221b7f923e926f4690ff8569da3", + "sha256:b2c00ad31fbc2cbac85d7d0fcf90853b2ca2e69d825a2d3f3edb842ef1544a2c", + "sha256:b4c153863dd6569f6511845922c53e39c8d61f6e81f228ad5443e690fca403de", + "sha256:b4f3d66dd0354b79761481fc15bdafaba0b9d9076f1f42cc9ce10d7fcbda205a", + "sha256:b99aac6bb2c37db336fa03a39b40ed4ef2818bf2dfb9441458165ebe88b793af", + "sha256:b9f6392d98c0bd70676ae41474e2eecf4c7150cb419237a41f8f96043fcb81d1", + "sha256:c537da54ce4ff7c15e78ab1292e5799d0d43a2108e006578a57f531866f64025", + "sha256:ca23db5fb195b5ef4fd1f77ce26cadefdf13dba71dab14dadd29b34d457d7c44", + "sha256:cc826b9a8176e686b67aa60fd6c6a7047b0461cae5591ea1dc73d28f72332a8a", + "sha256:cca83a629f77402cfadd58352e394d79a61c8015f1694b83ab72237ec3941f88", + "sha256:cf8d370b2fea27fb300825ec3984334f7dd54a581bde6456799ba3776915a656", + "sha256:d1175b0e0d6037fab207f05774a176d71210ebd40b1c51f480a04b65ec5c786d", + "sha256:d1996ee1330e245cd3aeda0887b4409e3930524c27642b046e4fae88ffa66c5e", + "sha256:d5a36953389f35f0a4e88dc796048829a2f467c9197265504593f0e420571547", + "sha256:da51d8928ad8b4244926fe862ba1795f0b6e68ed8c42cd2f822d435db9c2a8f4", + "sha256:e16e7297f29a544f49340012d6fc08cf14de0ab361c9eb7529f6a57a30cbfda1", + "sha256:e25b11a0417475f093d0f0809a149aff3943c2c56da50fdf2c3c88d57fe3dfbd", + "sha256:e4371591e621579cb6da8401e4ea405b33ff25a755874a3567c4075ca63d56e2", + "sha256:e653d36b1bf48fa78c7fcebb5fa679342e025121ace8c87ab05c1cefd33b34fc", + "sha256:e7d91a230c7f8af86c904a5a992b8c064b66330544693fd6759c3d6162382ecf", + "sha256:e851e6363d0dbe515d8de81fd544a2c956fdec6f8a049739562286727d4a00c3", + "sha256:ef7d48207926edbf8b16b336f779c557dd8f5a33035a85db9c4b0febb0706817", + "sha256:f7716f7e7138252d88607228ce40be22660d6608d20fd365d596e7ca0738e019", + "sha256:facaf11f21f3a4c51b62931feb13310e6fe3475f85e20d9c9fdce0d2ea561b87" + ], + "markers": "python_version >= '3.9'", + "version": "==6.2.0" }, "oauth2client": { "hashes": [ @@ -865,19 +880,123 @@ }, "pbr": { "hashes": [ - "sha256:788183e382e3d1d7707db08978239965e8b9e4e5ed42669bf4758186734d5f24", - "sha256:a776ae228892d8013649c0aeccbb3d5f99ee15e005a4cbb7e61d55a067b28a2a" + "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76", + "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b" ], "markers": "python_version >= '2.6'", - "version": "==6.1.0" + "version": "==6.1.1" + }, + "propcache": { + "hashes": [ + "sha256:050b571b2e96ec942898f8eb46ea4bfbb19bd5502424747e83badc2d4a99a44e", + "sha256:05543250deac8e61084234d5fc54f8ebd254e8f2b39a16b1dce48904f45b744b", + "sha256:069e7212890b0bcf9b2be0a03afb0c2d5161d91e1bf51569a64f629acc7defbf", + "sha256:09400e98545c998d57d10035ff623266927cb784d13dd2b31fd33b8a5316b85b", + "sha256:0c3c3a203c375b08fd06a20da3cf7aac293b834b6f4f4db71190e8422750cca5", + "sha256:0c86e7ceea56376216eba345aa1fc6a8a6b27ac236181f840d1d7e6a1ea9ba5c", + "sha256:0fbe94666e62ebe36cd652f5fc012abfbc2342de99b523f8267a678e4dfdee3c", + "sha256:17d1c688a443355234f3c031349da69444be052613483f3e4158eef751abcd8a", + "sha256:19a06db789a4bd896ee91ebc50d059e23b3639c25d58eb35be3ca1cbe967c3bf", + "sha256:1c5c7ab7f2bb3f573d1cb921993006ba2d39e8621019dffb1c5bc94cdbae81e8", + "sha256:1eb34d90aac9bfbced9a58b266f8946cb5935869ff01b164573a7634d39fbcb5", + "sha256:1f6cc0ad7b4560e5637eb2c994e97b4fa41ba8226069c9277eb5ea7101845b42", + "sha256:27c6ac6aa9fc7bc662f594ef380707494cb42c22786a558d95fcdedb9aa5d035", + "sha256:2d219b0dbabe75e15e581fc1ae796109b07c8ba7d25b9ae8d650da582bed01b0", + "sha256:2fce1df66915909ff6c824bbb5eb403d2d15f98f1518e583074671a30fe0c21e", + "sha256:319fa8765bfd6a265e5fa661547556da381e53274bc05094fc9ea50da51bfd46", + "sha256:359e81a949a7619802eb601d66d37072b79b79c2505e6d3fd8b945538411400d", + "sha256:3a02a28095b5e63128bcae98eb59025924f121f048a62393db682f049bf4ac24", + "sha256:3e19ea4ea0bf46179f8a3652ac1426e6dcbaf577ce4b4f65be581e237340420d", + "sha256:3e584b6d388aeb0001d6d5c2bd86b26304adde6d9bb9bfa9c4889805021b96de", + "sha256:40d980c33765359098837527e18eddefc9a24cea5b45e078a7f3bb5b032c6ecf", + "sha256:4114c4ada8f3181af20808bedb250da6bae56660e4b8dfd9cd95d4549c0962f7", + "sha256:43593c6772aa12abc3af7784bff4a41ffa921608dd38b77cf1dfd7f5c4e71371", + "sha256:47ef24aa6511e388e9894ec16f0fbf3313a53ee68402bc428744a367ec55b833", + "sha256:4cf9e93a81979f1424f1a3d155213dc928f1069d697e4353edb8a5eba67c6259", + "sha256:4d0dfdd9a2ebc77b869a0b04423591ea8823f791293b527dc1bb896c1d6f1136", + "sha256:563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25", + "sha256:58aa11f4ca8b60113d4b8e32d37e7e78bd8af4d1a5b5cb4979ed856a45e62005", + "sha256:5a0a9898fdb99bf11786265468571e628ba60af80dc3f6eb89a3545540c6b0ef", + "sha256:5aed8d8308215089c0734a2af4f2e95eeb360660184ad3912686c181e500b2e7", + "sha256:5b9145c35cc87313b5fd480144f8078716007656093d23059e8993d3a8fa730f", + "sha256:5cb5918253912e088edbf023788de539219718d3b10aef334476b62d2b53de53", + "sha256:5cdb0f3e1eb6dfc9965d19734d8f9c481b294b5274337a8cb5cb01b462dcb7e0", + "sha256:5ced33d827625d0a589e831126ccb4f5c29dfdf6766cac441d23995a65825dcb", + "sha256:603f1fe4144420374f1a69b907494c3acbc867a581c2d49d4175b0de7cc64566", + "sha256:61014615c1274df8da5991a1e5da85a3ccb00c2d4701ac6f3383afd3ca47ab0a", + "sha256:64a956dff37080b352c1c40b2966b09defb014347043e740d420ca1eb7c9b908", + "sha256:668ddddc9f3075af019f784456267eb504cb77c2c4bd46cc8402d723b4d200bf", + "sha256:6d8e309ff9a0503ef70dc9a0ebd3e69cf7b3894c9ae2ae81fc10943c37762458", + "sha256:6f173bbfe976105aaa890b712d1759de339d8a7cef2fc0a1714cc1a1e1c47f64", + "sha256:71ebe3fe42656a2328ab08933d420df5f3ab121772eef78f2dc63624157f0ed9", + "sha256:730178f476ef03d3d4d255f0c9fa186cb1d13fd33ffe89d39f2cda4da90ceb71", + "sha256:7d2d5a0028d920738372630870e7d9644ce437142197f8c827194fca404bf03b", + "sha256:7f30241577d2fef2602113b70ef7231bf4c69a97e04693bde08ddab913ba0ce5", + "sha256:813fbb8b6aea2fc9659815e585e548fe706d6f663fa73dff59a1677d4595a037", + "sha256:82de5da8c8893056603ac2d6a89eb8b4df49abf1a7c19d536984c8dd63f481d5", + "sha256:83be47aa4e35b87c106fc0c84c0fc069d3f9b9b06d3c494cd404ec6747544894", + "sha256:8638f99dca15b9dff328fb6273e09f03d1c50d9b6512f3b65a4154588a7595fe", + "sha256:87380fb1f3089d2a0b8b00f006ed12bd41bd858fabfa7330c954c70f50ed8757", + "sha256:88c423efef9d7a59dae0614eaed718449c09a5ac79a5f224a8b9664d603f04a3", + "sha256:89498dd49c2f9a026ee057965cdf8192e5ae070ce7d7a7bd4b66a8e257d0c976", + "sha256:8a17583515a04358b034e241f952f1715243482fc2c2945fd99a1b03a0bd77d6", + "sha256:916cd229b0150129d645ec51614d38129ee74c03293a9f3f17537be0029a9641", + "sha256:9532ea0b26a401264b1365146c440a6d78269ed41f83f23818d4b79497aeabe7", + "sha256:967a8eec513dbe08330f10137eacb427b2ca52118769e82ebcfcab0fba92a649", + "sha256:975af16f406ce48f1333ec5e912fe11064605d5c5b3f6746969077cc3adeb120", + "sha256:9979643ffc69b799d50d3a7b72b5164a2e97e117009d7af6dfdd2ab906cb72cd", + "sha256:9a8ecf38de50a7f518c21568c80f985e776397b902f1ce0b01f799aba1608b40", + "sha256:9cec3239c85ed15bfaded997773fdad9fb5662b0a7cbc854a43f291eb183179e", + "sha256:9e64e948ab41411958670f1093c0a57acfdc3bee5cf5b935671bbd5313bcf229", + "sha256:9f64d91b751df77931336b5ff7bafbe8845c5770b06630e27acd5dbb71e1931c", + "sha256:a0ab8cf8cdd2194f8ff979a43ab43049b1df0b37aa64ab7eca04ac14429baeb7", + "sha256:a110205022d077da24e60b3df8bcee73971be9575dec5573dd17ae5d81751111", + "sha256:a34aa3a1abc50740be6ac0ab9d594e274f59960d3ad253cd318af76b996dd654", + "sha256:a444192f20f5ce8a5e52761a031b90f5ea6288b1eef42ad4c7e64fef33540b8f", + "sha256:a461959ead5b38e2581998700b26346b78cd98540b5524796c175722f18b0294", + "sha256:a75801768bbe65499495660b777e018cbe90c7980f07f8aa57d6be79ea6f71da", + "sha256:aa8efd8c5adc5a2c9d3b952815ff8f7710cefdcaf5f2c36d26aff51aeca2f12f", + "sha256:aca63103895c7d960a5b9b044a83f544b233c95e0dcff114389d64d762017af7", + "sha256:b0313e8b923b3814d1c4a524c93dfecea5f39fa95601f6a9b1ac96cd66f89ea0", + "sha256:b23c11c2c9e6d4e7300c92e022046ad09b91fd00e36e83c44483df4afa990073", + "sha256:b303b194c2e6f171cfddf8b8ba30baefccf03d36a4d9cab7fd0bb68ba476a3d7", + "sha256:b655032b202028a582d27aeedc2e813299f82cb232f969f87a4fde491a233f11", + "sha256:bd39c92e4c8f6cbf5f08257d6360123af72af9f4da75a690bef50da77362d25f", + "sha256:bef100c88d8692864651b5f98e871fb090bd65c8a41a1cb0ff2322db39c96c27", + "sha256:c2fe5c910f6007e716a06d269608d307b4f36e7babee5f36533722660e8c4a70", + "sha256:c66d8ccbc902ad548312b96ed8d5d266d0d2c6d006fd0f66323e9d8f2dd49be7", + "sha256:cd6a55f65241c551eb53f8cf4d2f4af33512c39da5d9777694e9d9c60872f519", + "sha256:d249609e547c04d190e820d0d4c8ca03ed4582bcf8e4e160a6969ddfb57b62e5", + "sha256:d4e89cde74154c7b5957f87a355bb9c8ec929c167b59c83d90654ea36aeb6180", + "sha256:dc1915ec523b3b494933b5424980831b636fe483d7d543f7afb7b3bf00f0c10f", + "sha256:e1c4d24b804b3a87e9350f79e2371a705a188d292fd310e663483af6ee6718ee", + "sha256:e474fc718e73ba5ec5180358aa07f6aded0ff5f2abe700e3115c37d75c947e18", + "sha256:e4fe2a6d5ce975c117a6bb1e8ccda772d1e7029c1cca1acd209f91d30fa72815", + "sha256:e7fb9a84c9abbf2b2683fa3e7b0d7da4d8ecf139a1c635732a8bda29c5214b0e", + "sha256:e861ad82892408487be144906a368ddbe2dc6297074ade2d892341b35c59844a", + "sha256:ec314cde7314d2dd0510c6787326bbffcbdc317ecee6b7401ce218b3099075a7", + "sha256:ed5f6d2edbf349bd8d630e81f474d33d6ae5d07760c44d33cd808e2f5c8f4ae6", + "sha256:ef2e4e91fb3945769e14ce82ed53007195e616a63aa43b40fb7ebaaf907c8d4c", + "sha256:f011f104db880f4e2166bcdcf7f58250f7a465bc6b068dc84c824a3d4a5c94dc", + "sha256:f1528ec4374617a7a753f90f20e2f551121bb558fcb35926f99e3c42367164b8", + "sha256:f27785888d2fdd918bc36de8b8739f2d6c791399552333721b58193f68ea3e98", + "sha256:f35c7070eeec2cdaac6fd3fe245226ed2a6292d3ee8c938e5bb645b434c5f256", + "sha256:f3bbecd2f34d0e6d3c543fdb3b15d6b60dd69970c2b4c822379e5ec8f6f621d5", + "sha256:f6f1324db48f001c2ca26a25fa25af60711e09b9aaf4b28488602776f4f9a744", + "sha256:f78eb8422acc93d7b69964012ad7048764bb45a54ba7a39bb9e146c72ea29723", + "sha256:fb6e0faf8cb6b4beea5d6ed7b5a578254c6d7df54c36ccd3d8b3eb00d6770277", + "sha256:feccd282de1f6322f56f6845bf1207a537227812f0a9bf5571df52bb418d79d5" + ], + "markers": "python_version >= '3.9'", + "version": "==0.3.1" }, "proto-plus": { "hashes": [ - "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445", - "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12" + "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", + "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012" ], "markers": "python_version >= '3.7'", - "version": "==1.24.0" + "version": "==1.26.1" }, "protobuf": { "hashes": [ @@ -928,11 +1047,11 @@ }, "pyasn1-modules": { "hashes": [ - "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", - "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c" + "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", + "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" ], "markers": "python_version >= '3.8'", - "version": "==0.4.1" + "version": "==0.4.2" }, "pycparser": { "hashes": [ @@ -952,11 +1071,11 @@ }, "pyjwt": { "hashes": [ - "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850", - "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c" + "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", + "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb" ], - "markers": "python_version >= '3.8'", - "version": "==2.9.0" + "markers": "python_version >= '3.9'", + "version": "==2.10.1" }, "pymemcache": { "hashes": [ @@ -1070,11 +1189,11 @@ }, "redis": { "hashes": [ - "sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62", - "sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2" + "sha256:585dc516b9eb042a619ef0a39c3d7d55fe81bdb4df09a52c9cdde0d07bf1aa7d", + "sha256:e2b03db868160ee4591de3cb90d40ebb50a90dd302138775937f6a42b7ed183c" ], "index": "pypi", - "version": "==3.3.11" + "version": "==4.6.0" }, "requests": { "hashes": [ @@ -1118,19 +1237,19 @@ }, "setuptools": { "hashes": [ - "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2", - "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538" + "sha256:18fd474d4a82a5f83dac888df697af65afa82dec7323d09c3e37d1f14288da54", + "sha256:3e386e96793c8702ae83d17b853fb93d3e09ef82ec62722e61da5cd22376dcd8" ], - "markers": "python_version >= '3.8'", - "version": "==75.1.0" + "markers": "python_version >= '3.9'", + "version": "==78.1.0" }, "six": { "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" + "version": "==1.17.0" }, "uritemplate": { "hashes": [ @@ -1226,119 +1345,109 @@ }, "yarl": { "hashes": [ - "sha256:08d7148ff11cb8e886d86dadbfd2e466a76d5dd38c7ea8ebd9b0e07946e76e4b", - "sha256:098b870c18f1341786f290b4d699504e18f1cd050ed179af8123fd8232513424", - "sha256:11b3ca8b42a024513adce810385fcabdd682772411d95bbbda3b9ed1a4257644", - "sha256:1891d69a6ba16e89473909665cd355d783a8a31bc84720902c5911dbb6373465", - "sha256:1bbb418f46c7f7355084833051701b2301092e4611d9e392360c3ba2e3e69f88", - "sha256:1d0828e17fa701b557c6eaed5edbd9098eb62d8838344486248489ff233998b8", - "sha256:1d8e3ca29f643dd121f264a7c89f329f0fcb2e4461833f02de6e39fef80f89da", - "sha256:1fa56f34b2236f5192cb5fceba7bbb09620e5337e0b6dfe2ea0ddbd19dd5b154", - "sha256:216a6785f296169ed52cd7dcdc2612f82c20f8c9634bf7446327f50398732a51", - "sha256:22b739f99c7e4787922903f27a892744189482125cc7b95b747f04dd5c83aa9f", - "sha256:2430cf996113abe5aee387d39ee19529327205cda975d2b82c0e7e96e5fdabdc", - "sha256:269c201bbc01d2cbba5b86997a1e0f73ba5e2f471cfa6e226bcaa7fd664b598d", - "sha256:298c1eecfd3257aa16c0cb0bdffb54411e3e831351cd69e6b0739be16b1bdaa8", - "sha256:2a93a4557f7fc74a38ca5a404abb443a242217b91cd0c4840b1ebedaad8919d4", - "sha256:2b2442a415a5f4c55ced0fade7b72123210d579f7d950e0b5527fc598866e62c", - "sha256:2db874dd1d22d4c2c657807562411ffdfabec38ce4c5ce48b4c654be552759dc", - "sha256:309c104ecf67626c033845b860d31594a41343766a46fa58c3309c538a1e22b2", - "sha256:31497aefd68036d8e31bfbacef915826ca2e741dbb97a8d6c7eac66deda3b606", - "sha256:373f16f38721c680316a6a00ae21cc178e3a8ef43c0227f88356a24c5193abd6", - "sha256:396e59b8de7e4d59ff5507fb4322d2329865b909f29a7ed7ca37e63ade7f835c", - "sha256:3bb83a0f12701c0b91112a11148b5217617982e1e466069d0555be9b372f2734", - "sha256:3de86547c820e4f4da4606d1c8ab5765dd633189791f15247706a2eeabc783ae", - "sha256:3fdbf0418489525231723cdb6c79e7738b3cbacbaed2b750cb033e4ea208f220", - "sha256:40c6e73c03a6befb85b72da213638b8aaa80fe4136ec8691560cf98b11b8ae6e", - "sha256:44a4c40a6f84e4d5955b63462a0e2a988f8982fba245cf885ce3be7618f6aa7d", - "sha256:44b07e1690f010c3c01d353b5790ec73b2f59b4eae5b0000593199766b3f7a5c", - "sha256:45d23c4668d4925688e2ea251b53f36a498e9ea860913ce43b52d9605d3d8177", - "sha256:45f209fb4bbfe8630e3d2e2052535ca5b53d4ce2d2026bed4d0637b0416830da", - "sha256:4afdf84610ca44dcffe8b6c22c68f309aff96be55f5ea2fa31c0c225d6b83e23", - "sha256:4feaaa4742517eaceafcbe74595ed335a494c84634d33961214b278126ec1485", - "sha256:576365c9f7469e1f6124d67b001639b77113cfd05e85ce0310f5f318fd02fe85", - "sha256:5820bd4178e6a639b3ef1db8b18500a82ceab6d8b89309e121a6859f56585b05", - "sha256:5989a38ba1281e43e4663931a53fbf356f78a0325251fd6af09dd03b1d676a09", - "sha256:5a9bacedbb99685a75ad033fd4de37129449e69808e50e08034034c0bf063f99", - "sha256:5b66c87da3c6da8f8e8b648878903ca54589038a0b1e08dde2c86d9cd92d4ac9", - "sha256:5c5e32fef09ce101fe14acd0f498232b5710effe13abac14cd95de9c274e689e", - "sha256:658e8449b84b92a4373f99305de042b6bd0d19bf2080c093881e0516557474a5", - "sha256:6a2acde25be0cf9be23a8f6cbd31734536a264723fca860af3ae5e89d771cd71", - "sha256:6a5185ad722ab4dd52d5fb1f30dcc73282eb1ed494906a92d1a228d3f89607b0", - "sha256:6b7f6e699304717fdc265a7e1922561b02a93ceffdaefdc877acaf9b9f3080b8", - "sha256:703b0f584fcf157ef87816a3c0ff868e8c9f3c370009a8b23b56255885528f10", - "sha256:7055bbade838d68af73aea13f8c86588e4bcc00c2235b4b6d6edb0dbd174e246", - "sha256:78f271722423b2d4851cf1f4fa1a1c4833a128d020062721ba35e1a87154a049", - "sha256:7addd26594e588503bdef03908fc207206adac5bd90b6d4bc3e3cf33a829f57d", - "sha256:81bad32c8f8b5897c909bf3468bf601f1b855d12f53b6af0271963ee67fff0d2", - "sha256:82e692fb325013a18a5b73a4fed5a1edaa7c58144dc67ad9ef3d604eccd451ad", - "sha256:84bbcdcf393139f0abc9f642bf03f00cac31010f3034faa03224a9ef0bb74323", - "sha256:86c438ce920e089c8c2388c7dcc8ab30dfe13c09b8af3d306bcabb46a053d6f7", - "sha256:8be8cdfe20787e6a5fcbd010f8066227e2bb9058331a4eccddec6c0db2bb85b2", - "sha256:8c723c91c94a3bc8033dd2696a0f53e5d5f8496186013167bddc3fb5d9df46a3", - "sha256:8ca53632007c69ddcdefe1e8cbc3920dd88825e618153795b57e6ebcc92e752a", - "sha256:8f722f30366474a99745533cc4015b1781ee54b08de73260b2bbe13316079851", - "sha256:942c80a832a79c3707cca46bd12ab8aa58fddb34b1626d42b05aa8f0bcefc206", - "sha256:94a993f976cdcb2dc1b855d8b89b792893220db8862d1a619efa7451817c836b", - "sha256:95c6737f28069153c399d875317f226bbdea939fd48a6349a3b03da6829fb550", - "sha256:9915300fe5a0aa663c01363db37e4ae8e7c15996ebe2c6cce995e7033ff6457f", - "sha256:9a18595e6a2ee0826bf7dfdee823b6ab55c9b70e8f80f8b77c37e694288f5de1", - "sha256:9c8854b9f80693d20cec797d8e48a848c2fb273eb6f2587b57763ccba3f3bd4b", - "sha256:9cec42a20eae8bebf81e9ce23fb0d0c729fc54cf00643eb251ce7c0215ad49fe", - "sha256:9d2e1626be8712333a9f71270366f4a132f476ffbe83b689dd6dc0d114796c74", - "sha256:9d74f3c335cfe9c21ea78988e67f18eb9822f5d31f88b41aec3a1ec5ecd32da5", - "sha256:9fb4134cc6e005b99fa29dbc86f1ea0a298440ab6b07c6b3ee09232a3b48f495", - "sha256:a0ae6637b173d0c40b9c1462e12a7a2000a71a3258fa88756a34c7d38926911c", - "sha256:a31d21089894942f7d9a8df166b495101b7258ff11ae0abec58e32daf8088813", - "sha256:a3442c31c11088e462d44a644a454d48110f0588de830921fd201060ff19612a", - "sha256:ab9524e45ee809a083338a749af3b53cc7efec458c3ad084361c1dbf7aaf82a2", - "sha256:b1481c048fe787f65e34cb06f7d6824376d5d99f1231eae4778bbe5c3831076d", - "sha256:b8c837ab90c455f3ea8e68bee143472ee87828bff19ba19776e16ff961425b57", - "sha256:bbf2c3f04ff50f16404ce70f822cdc59760e5e2d7965905f0e700270feb2bbfc", - "sha256:bbf9c2a589be7414ac4a534d54e4517d03f1cbb142c0041191b729c2fa23f320", - "sha256:bcd5bf4132e6a8d3eb54b8d56885f3d3a38ecd7ecae8426ecf7d9673b270de43", - "sha256:c14c16831b565707149c742d87a6203eb5597f4329278446d5c0ae7a1a43928e", - "sha256:c49f3e379177f4477f929097f7ed4b0622a586b0aa40c07ac8c0f8e40659a1ac", - "sha256:c92b89bffc660f1274779cb6fbb290ec1f90d6dfe14492523a0667f10170de26", - "sha256:cd66152561632ed4b2a9192e7f8e5a1d41e28f58120b4761622e0355f0fe034c", - "sha256:cf1ad338620249f8dd6d4b6a91a69d1f265387df3697ad5dc996305cf6c26fb2", - "sha256:d07b52c8c450f9366c34aa205754355e933922c79135125541daae6cbf31c799", - "sha256:d0d12fe78dcf60efa205e9a63f395b5d343e801cf31e5e1dda0d2c1fb618073d", - "sha256:d4ee1d240b84e2f213565f0ec08caef27a0e657d4c42859809155cf3a29d1735", - "sha256:d959fe96e5c2712c1876d69af0507d98f0b0e8d81bee14cfb3f6737470205419", - "sha256:dcaef817e13eafa547cdfdc5284fe77970b891f731266545aae08d6cce52161e", - "sha256:df4e82e68f43a07735ae70a2d84c0353e58e20add20ec0af611f32cd5ba43fb4", - "sha256:ec8cfe2295f3e5e44c51f57272afbd69414ae629ec7c6b27f5a410efc78b70a0", - "sha256:ec9dd328016d8d25702a24ee274932aebf6be9787ed1c28d021945d264235b3c", - "sha256:ef9b85fa1bc91c4db24407e7c4da93a5822a73dd4513d67b454ca7064e8dc6a3", - "sha256:f3bf60444269345d712838bb11cc4eadaf51ff1a364ae39ce87a5ca8ad3bb2c8", - "sha256:f452cc1436151387d3d50533523291d5f77c6bc7913c116eb985304abdbd9ec9", - "sha256:f7917697bcaa3bc3e83db91aa3a0e448bf5cde43c84b7fc1ae2427d2417c0224", - "sha256:f90575e9fe3aae2c1e686393a9689c724cd00045275407f71771ae5d690ccf38", - "sha256:fb382fd7b4377363cc9f13ba7c819c3c78ed97c36a82f16f3f92f108c787cbbf", - "sha256:fb9f59f3848edf186a76446eb8bcf4c900fe147cb756fbbd730ef43b2e67c6a7", - "sha256:fc2931ac9ce9c61c9968989ec831d3a5e6fcaaff9474e7cfa8de80b7aff5a093" - ], - "markers": "python_version >= '3.8'", - "version": "==1.13.1" + "sha256:00e5a1fea0fd4f5bfa7440a47eff01d9822a65b4488f7cff83155a0f31a2ecba", + "sha256:02ddb6756f8f4517a2d5e99d8b2f272488e18dd0bfbc802f31c16c6c20f22193", + "sha256:045b8482ce9483ada4f3f23b3774f4e1bf4f23a2d5c912ed5170f68efb053318", + "sha256:09c7907c8548bcd6ab860e5f513e727c53b4a714f459b084f6580b49fa1b9cee", + "sha256:0b0cad37311123211dc91eadcb322ef4d4a66008d3e1bdc404808992260e1a0e", + "sha256:0b3c92fa08759dbf12b3a59579a4096ba9af8dd344d9a813fc7f5070d86bbab1", + "sha256:0fb2171a4486bb075316ee754c6d8382ea6eb8b399d4ec62fde2b591f879778a", + "sha256:1a74a13a4c857a84a845505fd2d68e54826a2cd01935a96efb1e9d86c728e186", + "sha256:1d407181cfa6e70077df3377938c08012d18893f9f20e92f7d2f314a437c30b1", + "sha256:1dd4bdd05407ced96fed3d7f25dbbf88d2ffb045a0db60dbc247f5b3c5c25d50", + "sha256:25b411eddcfd56a2f0cd6a384e9f4f7aa3efee14b188de13048c25b5e91f1640", + "sha256:2d06d3005e668744e11ed80812e61efd77d70bb7f03e33c1598c301eea20efbb", + "sha256:2ec9bbba33b2d00999af4631a3397d1fd78290c48e2a3e52d8dd72db3a067ac8", + "sha256:3236da9272872443f81fedc389bace88408f64f89f75d1bdb2256069a8730ccc", + "sha256:35098b24e0327fc4ebdc8ffe336cee0a87a700c24ffed13161af80124b7dc8e5", + "sha256:41f7ce59d6ee7741af71d82020346af364949314ed3d87553763a2df1829cc58", + "sha256:436c4fc0a4d66b2badc6c5fc5ef4e47bb10e4fd9bf0c79524ac719a01f3607c2", + "sha256:4891ed92157e5430874dad17b15eb1fda57627710756c27422200c52d8a4e393", + "sha256:4ac515b860c36becb81bb84b667466885096b5fc85596948548b667da3bf9f24", + "sha256:5094d9206c64181d0f6e76ebd8fb2f8fe274950a63890ee9e0ebfd58bf9d787b", + "sha256:54d6921f07555713b9300bee9c50fb46e57e2e639027089b1d795ecd9f7fa910", + "sha256:578e281c393af575879990861823ef19d66e2b1d0098414855dd367e234f5b3c", + "sha256:5a3f356548e34a70b0172d8890006c37be92995f62d95a07b4a42e90fba54272", + "sha256:602d98f2c2d929f8e697ed274fbadc09902c4025c5a9963bf4e9edfc3ab6f7ed", + "sha256:61b1a825a13bef4a5f10b1885245377d3cd0bf87cba068e1d9a88c2ae36880e1", + "sha256:61e5e68cb65ac8f547f6b5ef933f510134a6bf31bb178be428994b0cb46c2a04", + "sha256:61ee62ead9b68b9123ec24bc866cbef297dd266175d53296e2db5e7f797f902d", + "sha256:6333c5a377c8e2f5fae35e7b8f145c617b02c939d04110c76f29ee3676b5f9a5", + "sha256:6748dbf9bfa5ba1afcc7556b71cda0d7ce5f24768043a02a58846e4a443d808d", + "sha256:67a283dd2882ac98cc6318384f565bffc751ab564605959df4752d42483ad889", + "sha256:75674776d96d7b851b6498f17824ba17849d790a44d282929c42dbb77d4f17ae", + "sha256:757e81cae69244257d125ff31663249b3013b5dc0a8520d73694aed497fb195b", + "sha256:77a6e85b90a7641d2e07184df5557132a337f136250caafc9ccaa4a2a998ca2c", + "sha256:7c33dd1931a95e5d9a772d0ac5e44cac8957eaf58e3c8da8c1414de7dd27c576", + "sha256:7df647e8edd71f000a5208fe6ff8c382a1de8edfbccdbbfe649d263de07d8c34", + "sha256:7e2ee16578af3b52ac2f334c3b1f92262f47e02cc6193c598502bd46f5cd1477", + "sha256:80316a8bd5109320d38eef8833ccf5f89608c9107d02d2a7f985f98ed6876990", + "sha256:82123d0c954dc58db301f5021a01854a85bf1f3bb7d12ae0c01afc414a882ca2", + "sha256:84b2deecba4a3f1a398df819151eb72d29bfeb3b69abb145a00ddc8d30094512", + "sha256:8503ad47387b8ebd39cbbbdf0bf113e17330ffd339ba1144074da24c545f0069", + "sha256:877d209b6aebeb5b16c42cbb377f5f94d9e556626b1bfff66d7b0d115be88d0a", + "sha256:8874027a53e3aea659a6d62751800cf6e63314c160fd607489ba5c2edd753cf6", + "sha256:88a19f62ff30117e706ebc9090b8ecc79aeb77d0b1f5ec10d2d27a12bc9f66d0", + "sha256:8d39d351e7faf01483cc7ff7c0213c412e38e5a340238826be7e0e4da450fdc8", + "sha256:90adb47ad432332d4f0bc28f83a5963f426ce9a1a8809f5e584e704b82685dcb", + "sha256:913829534200eb0f789d45349e55203a091f45c37a2674678744ae52fae23efa", + "sha256:93b2e109287f93db79210f86deb6b9bbb81ac32fc97236b16f7433db7fc437d8", + "sha256:9d41beda9dc97ca9ab0b9888cb71f7539124bc05df02c0cff6e5acc5a19dcc6e", + "sha256:a440a2a624683108a1b454705ecd7afc1c3438a08e890a1513d468671d90a04e", + "sha256:a4bb030cf46a434ec0225bddbebd4b89e6471814ca851abb8696170adb163985", + "sha256:a9ca04806f3be0ac6d558fffc2fdf8fcef767e0489d2684a21912cc4ed0cd1b8", + "sha256:ac1801c45cbf77b6c99242eeff4fffb5e4e73a800b5c4ad4fc0be5def634d2e1", + "sha256:ac36703a585e0929b032fbaab0707b75dc12703766d0b53486eabd5139ebadd5", + "sha256:b1771de9944d875f1b98a745bc547e684b863abf8f8287da8466cf470ef52690", + "sha256:b464c4ab4bfcb41e3bfd3f1c26600d038376c2de3297760dfe064d2cb7ea8e10", + "sha256:b4f6450109834af88cb4cc5ecddfc5380ebb9c228695afc11915a0bf82116789", + "sha256:b57f4f58099328dfb26c6a771d09fb20dbbae81d20cfb66141251ea063bd101b", + "sha256:b643562c12680b01e17239be267bc306bbc6aac1f34f6444d1bded0c5ce438ca", + "sha256:b958ddd075ddba5b09bb0be8a6d9906d2ce933aee81100db289badbeb966f54e", + "sha256:b9d60031cf568c627d028239693fd718025719c02c9f55df0a53e587aab951b5", + "sha256:ba23302c0c61a9999784e73809427c9dbedd79f66a13d84ad1b1943802eaaf59", + "sha256:ba87babd629f8af77f557b61e49e7c7cac36f22f871156b91e10a6e9d4f829e9", + "sha256:c017a3b6df3a1bd45b9fa49a0f54005e53fbcad16633870104b66fa1a30a29d8", + "sha256:c1e1cc06da1491e6734f0ea1e6294ce00792193c463350626571c287c9a704db", + "sha256:c654d5207c78e0bd6d749f6dae1dcbbfde3403ad3a4b11f3c5544d9906969dde", + "sha256:c69697d3adff5aa4f874b19c0e4ed65180ceed6318ec856ebc423aa5850d84f7", + "sha256:c7d79f7d9aabd6011004e33b22bc13056a3e3fb54794d138af57f5ee9d9032cb", + "sha256:ccaa3a4b521b780a7e771cc336a2dba389a0861592bbce09a476190bb0c8b4b3", + "sha256:ccd17349166b1bee6e529b4add61727d3f55edb7babbe4069b5764c9587a8cc6", + "sha256:ce1af883b94304f493698b00d0f006d56aea98aeb49d75ec7d98cd4a777e9285", + "sha256:d0e883008013c0e4aef84dcfe2a0b172c4d23c2669412cf5b3371003941f72bb", + "sha256:d980e0325b6eddc81331d3f4551e2a333999fb176fd153e075c6d1c2530aa8a8", + "sha256:e17c9361d46a4d5addf777c6dd5eab0715a7684c2f11b88c67ac37edfba6c482", + "sha256:e2c08cc9b16f4f4bc522771d96734c7901e7ebef70c6c5c35dd0f10845270bcd", + "sha256:e35ef8683211db69ffe129a25d5634319a677570ab6b2eba4afa860f54eeaf75", + "sha256:e3b9fd71836999aad54084906f8663dffcd2a7fb5cdafd6c37713b2e72be1760", + "sha256:ef9f7768395923c3039055c14334ba4d926f3baf7b776c923c93d80195624782", + "sha256:f52a265001d830bc425f82ca9eabda94a64a4d753b07d623a9f2863fde532b53", + "sha256:f91c4803173928a25e1a55b943c81f55b8872f0018be83e3ad4938adffb77dd2", + "sha256:fbd6748e8ab9b41171bb95c6142faf068f5ef1511935a0aa07025438dd9a9bc1", + "sha256:fe57328fbc1bfd0bd0514470ac692630f3901c0ee39052ae47acd1d90a436719", + "sha256:fea09ca13323376a2fdfb353a5fa2e59f90cd18d7ca4eaa1fd31f0a8b4f91e62" + ], + "markers": "python_version >= '3.9'", + "version": "==1.18.3" } }, "develop": { "cachecontrol": { "hashes": [ - "sha256:7db1195b41c81f8274a7bbd97c956f44e8348265a1bc7641c37dfebc39f0c938", - "sha256:f5bf3f0620c38db2e5122c0726bdebb0d16869de966ea6a2befe92470b740ea0" + "sha256:7d47d19f866409b98ff6025b6a0fca8e4c791fb31abbd95f622093894ce903a2", + "sha256:ebad2091bf12d0d200dfc2464330db638c5deb41d546f6d7aca079e87290f3b0" ], - "markers": "python_version >= '3.7'", - "version": "==0.14.0" + "markers": "python_version >= '3.8'", + "version": "==0.14.2" }, "cachetools": { "hashes": [ - "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292", - "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a" + "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", + "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" ], "markers": "python_version >= '3.7'", - "version": "==5.5.0" + "version": "==5.5.2" }, "certifi": { "hashes": [ @@ -1516,11 +1625,11 @@ }, "click": { "hashes": [ - "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de" + "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", + "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a" ], "markers": "python_version >= '3.7'", - "version": "==8.1.7" + "version": "==8.1.8" }, "cryptography": { "hashes": [ @@ -1607,11 +1716,11 @@ }, "google-cloud-firestore": { "hashes": [ - "sha256:1b2ce6e0b791aee89a1e4f072beba1012247e89baca361eed721fb467fe054b0", - "sha256:b49f0019d7bd0d4ab5972a4cff13994b0aabe72d24242200d904db2fb49df7f7" + "sha256:5970e1e6b9fd1c2661fa91d4bb6160110699a2456ec4d08c33199aca4aab28a0", + "sha256:6cae93e9b23fd1f5cf19dcc7417ba34712388a41692158d4396c63309b287779" ], "markers": "platform_python_implementation != 'PyPy'", - "version": "==2.19.0" + "version": "==2.20.1" }, "google-cloud-storage": { "hashes": [ @@ -1705,11 +1814,11 @@ }, "googleapis-common-protos": { "hashes": [ - "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63", - "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0" + "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212", + "sha256:3e1b904a27a33c821b4b749fd31d334c0c9c30e6113023d495e48979a3dc9c5f" ], "markers": "python_version >= '3.7'", - "version": "==1.65.0" + "version": "==1.69.2" }, "grpcio": { "hashes": [ @@ -1811,69 +1920,70 @@ }, "markupsafe": { "hashes": [ - "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", - "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", - "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", - "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", - "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", - "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", - "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", - "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", - "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", - "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", - "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", - "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", - "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", - "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", - "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", - "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", - "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", - "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", - "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", - "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", - "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", - "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", - "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", - "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", - "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", - "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", - "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", - "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", - "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", - "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", - "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", - "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", - "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", - "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", - "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", - "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", - "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", - "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", - "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", - "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", - "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", - "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", - "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", - "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", - "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", - "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", - "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", - "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", - "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", - "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", - "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", - "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", - "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", - "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", - "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", - "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", - "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", - "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", - "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", - "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.5" + "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", + "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", + "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0", + "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", + "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", + "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13", + "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", + "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", + "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", + "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", + "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0", + "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", + "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", + "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", + "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", + "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff", + "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", + "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", + "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", + "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", + "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", + "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", + "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", + "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", + "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a", + "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", + "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", + "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", + "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", + "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144", + "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f", + "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", + "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", + "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", + "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", + "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", + "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", + "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", + "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", + "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", + "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", + "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", + "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", + "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", + "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", + "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", + "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", + "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", + "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29", + "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", + "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", + "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", + "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", + "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", + "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", + "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a", + "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178", + "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", + "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", + "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", + "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50" + ], + "markers": "python_version >= '3.9'", + "version": "==3.0.2" }, "msgpack": { "hashes": [ @@ -1947,11 +2057,11 @@ }, "proto-plus": { "hashes": [ - "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445", - "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12" + "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", + "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012" ], "markers": "python_version >= '3.7'", - "version": "==1.24.0" + "version": "==1.26.1" }, "protobuf": { "hashes": [ @@ -1982,11 +2092,11 @@ }, "pyasn1-modules": { "hashes": [ - "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", - "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c" + "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", + "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" ], "markers": "python_version >= '3.8'", - "version": "==0.4.1" + "version": "==0.4.2" }, "pycparser": { "hashes": [ @@ -1998,11 +2108,11 @@ }, "pyjwt": { "hashes": [ - "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850", - "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c" + "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", + "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb" ], - "markers": "python_version >= '3.8'", - "version": "==2.9.0" + "markers": "python_version >= '3.9'", + "version": "==2.10.1" }, "pyparsing": { "hashes": [ diff --git a/src/appengine/index.yaml b/src/appengine/index.yaml index df56b595e5..3f67327037 100644 --- a/src/appengine/index.yaml +++ b/src/appengine/index.yaml @@ -588,3 +588,10 @@ indexes: properties: - name: keywords - name: bot_name + +- kind: WindowRateLimitTask + properties: + - name: job_name + - name: task_argument + - name: task_name + - name: timestamp \ No newline at end of file diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index bdacfb063d..a0ae77b249 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -998,7 +998,7 @@ def is_oss_fuzz(): def is_chromium(): """If this is an instance of chromium fuzzing.""" - return default_project_name() in ('chromium', 'chromium-testing') + return default_project_name() in ('chromium', 'chromium-testing', 'clusterfuzz-staging') def file_hash(file_path): diff --git a/src/clusterfuzz/_internal/bot/tasks/commands.py b/src/clusterfuzz/_internal/bot/tasks/commands.py index 6f76087a85..6f83bb43b9 100644 --- a/src/clusterfuzz/_internal/bot/tasks/commands.py +++ b/src/clusterfuzz/_internal/bot/tasks/commands.py @@ -277,10 +277,6 @@ def process_command(task): task.high_end, task.is_command_override) -def _get_task_id(task_name, task_argument, job_name): - return f'{task_name},{task_argument},{job_name},{uuid.uuid4()}' - - # pylint: disable=too-many-nested-blocks # TODO(mbarbella): Rewrite this function to avoid nesting issues. @set_task_payload @@ -296,8 +292,11 @@ def process_command_impl(task_name, task_argument, job_name, high_end, # "postprocess". task_id = None else: - task_id = _get_task_id(task_name, task_argument, job_name) + task_id = uuid.uuid4() environment.set_value('CF_TASK_ID', task_id) + environment.set_value('CF_TASK_NAME', task_name) + environment.set_value('CF_TASK_ARGUMENT', task_argument) + environment.set_value('CF_TASK_JOB_NAME', job_name) if job_name != 'none': job = data_types.Job.query(data_types.Job.name == job_name).get() # Job might be removed. In that case, we don't want an exception @@ -451,6 +450,9 @@ def process_command_impl(task_name, task_argument, job_name, high_end, uworker_env['TASK_ARGUMENT'] = task_argument uworker_env['JOB_NAME'] = job_name uworker_env['CF_TASK_ID'] = task_id + uworker_env['CF_TASK_NAME'] = task_name + uworker_env['CF_TASK_ARGUMENT'] = task_argument + uworker_env['CF_TASK_JOB_NAME'] = job_name # Match the cpu architecture with the ones required in the job definition. # If they don't match, then bail out and recreate task. @@ -474,5 +476,9 @@ def process_command_impl(task_name, task_argument, job_name, high_end, finally: # Final clean up. cleanup_task_state() - if 'CF_TASK_ID' in os.environ: - del os.environ['CF_TASK_ID'] + tear_down_envs = [ + 'CF_TASK_ID', 'CF_TASK_NAME', 'CF_TASK_ARGUMENT', 'CF_TASK_JOB_NAME' + ] + for env_key in tear_down_envs: + if env_key in os.environ: + del os.environ[env_key] diff --git a/src/clusterfuzz/_internal/bot/tasks/setup.py b/src/clusterfuzz/_internal/bot/tasks/setup.py index 0a46472bf2..6a43eb32e5 100644 --- a/src/clusterfuzz/_internal/bot/tasks/setup.py +++ b/src/clusterfuzz/_internal/bot/tasks/setup.py @@ -495,6 +495,7 @@ def update_data_bundle( logs.info('Data bundles: normal path.') result = corpus_manager.sync_data_bundle_corpus_to_disk( data_bundle_corpus, data_bundle_directory) + logs.info(f'Result = {result}') else: logs.info('Data bundles: untrusted runner path.') from clusterfuzz._internal.bot.untrusted_runner import \ @@ -509,6 +510,7 @@ def update_data_bundle( data_bundle_corpus.gcs_url, worker_data_bundle_directory, delete=False) + logs.info(f'Result = {result}') result = result.return_code == 0 if not result: diff --git a/src/clusterfuzz/_internal/bot/tasks/task_types.py b/src/clusterfuzz/_internal/bot/tasks/task_types.py index 39e83a536b..c64f7f9671 100644 --- a/src/clusterfuzz/_internal/bot/tasks/task_types.py +++ b/src/clusterfuzz/_internal/bot/tasks/task_types.py @@ -146,6 +146,7 @@ def execute(self, task_argument, job_type, uworker_env): assert swarming.is_swarming_task(command, job_type) swarming.push_swarming_task(command, download_url, job_type) + @logs.task_stage_context(logs.Stage.PREPROCESS) def preprocess(self, task_argument, job_type, uworker_env): result = utasks.tworker_preprocess(self.module, task_argument, job_type, uworker_env) @@ -170,6 +171,7 @@ def __init__(self, module): # many different tasks. super().__init__('none') + @logs.task_stage_context(logs.Stage.POSTPROCESS) def execute(self, task_argument, job_type, uworker_env): """Executes postprocessing of a utask.""" # These values are None for now. @@ -190,6 +192,7 @@ def __init__(self, module): del module super().__init__('none') + @logs.task_stage_context(logs.Stage.MAIN) def execute(self, task_argument, job_type, uworker_env): """Executes uworker_main of a utask.""" # These values are None for now. diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py index c4586dac5a..d09d36e9f9 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/__init__.py @@ -240,6 +240,7 @@ def _start_web_server_if_needed(job_type): logs.error('Failed to start web server, skipping.') +@logs.task_stage_context(logs.Stage.PREPROCESS) def tworker_preprocess_no_io(utask_module, task_argument, job_type, uworker_env): """Executes the preprocessing step of the utask |utask_module| and returns the @@ -253,6 +254,7 @@ def tworker_preprocess_no_io(utask_module, task_argument, job_type, return uworker_io.serialize_uworker_input(uworker_input) +@logs.task_stage_context(logs.Stage.MAIN) def uworker_main_no_io(utask_module, serialized_uworker_input): """Executes the main part of a utask on the uworker (locally if not using remote executor).""" @@ -283,6 +285,7 @@ def uworker_main_no_io(utask_module, serialized_uworker_input): # TODO(metzman): Stop passing module to this function and `uworker_main_no_io`. # Make them consistent with the I/O versions. +@logs.task_stage_context(logs.Stage.POSTPROCESS) def tworker_postprocess_no_io(utask_module, uworker_output, uworker_input): """Executes the postprocess step on the trusted (t)worker (in this case it is the same bot as the uworker).""" @@ -330,6 +333,7 @@ def set_uworker_env(uworker_env: dict) -> None: environment.set_value(key, value) +@logs.task_stage_context(logs.Stage.MAIN) def uworker_main(input_download_url) -> None: """Executes the main part of a utask on the uworker (locally if not using remote executor).""" @@ -380,6 +384,7 @@ def uworker_bot_main(): return 0 +@logs.task_stage_context(logs.Stage.POSTPROCESS) def tworker_postprocess(output_download_url) -> None: """Executes the postprocess step on the trusted (t)worker.""" logs.info('Starting postprocess untrusted worker.') diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py index 78c79f337d..5435853d40 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/progression_task.py @@ -436,38 +436,38 @@ def _set_regression_testcase_upload_url( def utask_preprocess(testcase_id, job_type, uworker_env): """Runs preprocessing for progression task.""" testcase = data_handler.get_testcase_by_id(testcase_id) - if not testcase: - return None - - if testcase.fixed: - logs.error(f'Fixed range is already set as {testcase.fixed}, skip.') - return None - - # Set a flag to indicate we are running progression task. This shows pending - # status on testcase report page and avoid conflicting testcase updates by - # triage cron. - testcase.set_metadata('progression_pending', True) - data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) - blob_name, blob_upload_url = blobs.get_blob_signed_upload_url() - progression_input = uworker_msg_pb2.ProgressionTaskInput( # pylint: disable=no-member - custom_binary=build_manager.is_custom_binary(), - bad_revisions=build_manager.get_job_bad_revisions(), - blob_name=blob_name, - stacktrace_upload_url=blob_upload_url) - # Setup testcase and its dependencies. - setup_input = setup.preprocess_setup_testcase(testcase, uworker_env) - - _set_regression_testcase_upload_url(progression_input, testcase) - uworker_input = uworker_msg_pb2.Input( # pylint: disable=no-member - job_type=job_type, - testcase_id=str(testcase_id), - uworker_env=uworker_env, - progression_task_input=progression_input, - testcase=uworker_io.entity_to_protobuf(testcase), - setup_input=setup_input) - - testcase_manager.preprocess_testcase_manager(testcase, uworker_input) - return uworker_input + with logs.progression_log_context(testcase, testcase.get_fuzz_target()): + if not testcase: + return None + if testcase.fixed: + logs.error(f'Fixed range is already set as {testcase.fixed}, skip.') + return None + + # Set a flag to indicate we are running progression task. This shows pending + # status on testcase report page and avoid conflicting testcase updates by + # triage cron. + testcase.set_metadata('progression_pending', True) + data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) + blob_name, blob_upload_url = blobs.get_blob_signed_upload_url() + progression_input = uworker_msg_pb2.ProgressionTaskInput( # pylint: disable=no-member + custom_binary=build_manager.is_custom_binary(), + bad_revisions=build_manager.get_job_bad_revisions(), + blob_name=blob_name, + stacktrace_upload_url=blob_upload_url) + # Setup testcase and its dependencies. + setup_input = setup.preprocess_setup_testcase(testcase, uworker_env) + + _set_regression_testcase_upload_url(progression_input, testcase) + uworker_input = uworker_msg_pb2.Input( # pylint: disable=no-member + job_type=job_type, + testcase_id=str(testcase_id), + uworker_env=uworker_env, + progression_task_input=progression_input, + testcase=uworker_io.entity_to_protobuf(testcase), + setup_input=setup_input) + + testcase_manager.preprocess_testcase_manager(testcase, uworker_input) + return uworker_input def find_fixed_range(uworker_input): @@ -684,8 +684,10 @@ def utask_main(uworker_input): """Executes the untrusted part of progression_task.""" testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) - uworker_io.check_handling_testcase_safe(testcase) - return find_fixed_range(uworker_input) + with logs.progression_log_context( + testcase, testcase_manager.get_fuzz_target_from_input(uworker_input)): + uworker_io.check_handling_testcase_safe(testcase) + return find_fixed_range(uworker_input) _ERROR_HANDLER = uworker_handle_errors.CompositeErrorHandler({ @@ -713,53 +715,54 @@ def utask_postprocess(output: uworker_msg_pb2.Output): # pylint: disable=no-mem """Trusted: Cleans up after a uworker execute_task, writing anything needed to the db.""" testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) - _maybe_clear_progression_last_min_max_metadata(testcase, output) - _cleanup_stacktrace_blob_from_storage(output) - task_output = None + with logs.progression_log_context(testcase, testcase.get_fuzz_target()): + _maybe_clear_progression_last_min_max_metadata(testcase, output) + _cleanup_stacktrace_blob_from_storage(output) + task_output = None - if output.issue_metadata: - _update_issue_metadata(testcase, json.loads(output.issue_metadata)) + if output.issue_metadata: + _update_issue_metadata(testcase, json.loads(output.issue_metadata)) - if output.HasField('progression_task_output'): - task_output = output.progression_task_output - _update_build_metadata(output.uworker_input.job_type, - task_output.build_data_list) + if output.HasField('progression_task_output'): + task_output = output.progression_task_output + _update_build_metadata(output.uworker_input.job_type, + task_output.build_data_list) - if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member - _ERROR_HANDLER.handle(output) - return + if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member + _ERROR_HANDLER.handle(output) + return - # If there is a fine grained bisection service available, request it. Both - # regression and fixed ranges are requested once. Regression is also requested - # here as the bisection service may require details that are not yet available - # (e.g. issue ID) at the time regress_task completes. - bisection.request_bisection(testcase) + # If there is a fine grained bisection service available, request it. + # Both regression and fixed ranges are requested once. Regression is also + # requested here as the bisection service may require details that + # are not yet available (e.g. issue ID) at the time regress_task completes. + bisection.request_bisection(testcase) - if task_output and task_output.crash_on_latest: - crash_on_latest(output) - return + if task_output and task_output.crash_on_latest: + crash_on_latest(output) + return - if output.uworker_input.progression_task_input.custom_binary: - # Retry once on another bot to confirm our results and in case this bot is - # in a bad state which we didn't catch through our usual means. - if data_handler.is_first_attempt_for_task( - 'progression', testcase, reset_after_retry=True): - tasks.add_task('progression', output.uworker_input.testcase_id, - output.uworker_input.job_type) + if output.uworker_input.progression_task_input.custom_binary: + # Retry once on another bot to confirm our results and in case this bot is + # in a bad state which we didn't catch through our usual means. + if data_handler.is_first_attempt_for_task( + 'progression', testcase, reset_after_retry=True): + tasks.add_task('progression', output.uworker_input.testcase_id, + output.uworker_input.job_type) + data_handler.update_progression_completion_metadata( + testcase, task_output.crash_revision) + return + + # The bug is fixed. + testcase.fixed = 'Yes' + testcase.open = False data_handler.update_progression_completion_metadata( - testcase, task_output.crash_revision) + testcase, + task_output.crash_revision, + message='fixed on latest custom build') return - # The bug is fixed. - testcase.fixed = 'Yes' - testcase.open = False - data_handler.update_progression_completion_metadata( - testcase, - task_output.crash_revision, - message='fixed on latest custom build') - return - - testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) - if task_output.HasField('min_revision'): - _save_fixed_range(output.uworker_input.testcase_id, - task_output.min_revision, task_output.max_revision) + testcase = data_handler.get_testcase_by_id(output.uworker_input.testcase_id) + if task_output.HasField('min_revision'): + _save_fixed_range(output.uworker_input.testcase_id, + task_output.min_revision, task_output.max_revision) diff --git a/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py b/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py index c636e46c32..f4eaea8c89 100644 --- a/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/utasks/regression_task.py @@ -588,36 +588,37 @@ def utask_preprocess(testcase_id: str, job_type: str, Runs on a trusted worker. """ testcase = data_handler.get_testcase_by_id(testcase_id) - - if testcase.regression: - logs.error( - f'Regression range is already set as {testcase.regression}, skip.') - return None - - # This task is not applicable for custom binaries. - if build_manager.is_custom_binary(): - testcase.regression = 'NA' - data_handler.update_testcase_comment(testcase, data_types.TaskState.ERROR, - 'Not applicable for custom binaries') - return None - - data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) - - setup_input = setup.preprocess_setup_testcase(testcase, uworker_env) - - task_input = uworker_msg_pb2.RegressionTaskInput( # pylint: disable=no-member - bad_revisions=build_manager.get_job_bad_revisions()) - - uworker_input = uworker_msg_pb2.Input( # pylint: disable=no-member - testcase_id=testcase_id, - testcase=uworker_io.entity_to_protobuf(testcase), - job_type=job_type, - uworker_env=uworker_env, - setup_input=setup_input, - regression_task_input=task_input, - ) - testcase_manager.preprocess_testcase_manager(testcase, uworker_input) - return uworker_input + with logs.regression_log_context(testcase, testcase.get_fuzz_target()): + if testcase.regression: + logs.error( + f'Regression range is already set as {testcase.regression}, skip.') + return None + + # This task is not applicable for custom binaries. + if build_manager.is_custom_binary(): + testcase.regression = 'NA' + data_handler.update_testcase_comment( + testcase, data_types.TaskState.ERROR, + 'Not applicable for custom binaries') + return None + + data_handler.update_testcase_comment(testcase, data_types.TaskState.STARTED) + + setup_input = setup.preprocess_setup_testcase(testcase, uworker_env) + + task_input = uworker_msg_pb2.RegressionTaskInput( # pylint: disable=no-member + bad_revisions=build_manager.get_job_bad_revisions()) + + uworker_input = uworker_msg_pb2.Input( # pylint: disable=no-member + testcase_id=testcase_id, + testcase=uworker_io.entity_to_protobuf(testcase), + job_type=job_type, + uworker_env=uworker_env, + setup_input=setup_input, + regression_task_input=task_input, + ) + testcase_manager.preprocess_testcase_manager(testcase, uworker_input) + return uworker_input def utask_main( @@ -629,8 +630,10 @@ def utask_main( """ testcase = uworker_io.entity_from_protobuf(uworker_input.testcase, data_types.Testcase) - uworker_io.check_handling_testcase_safe(testcase) - return find_regression_range(uworker_input) + with logs.regression_log_context( + testcase, testcase_manager.get_fuzz_target_from_input(uworker_input)): + uworker_io.check_handling_testcase_safe(testcase) + return find_regression_range(uworker_input) def handle_revision_list_error(output: uworker_msg_pb2.Output): # pylint: disable=no-member @@ -723,27 +726,31 @@ def utask_postprocess(output: uworker_msg_pb2.Output) -> None: # pylint: disabl Runs on a trusted worker. """ testcase_id = output.uworker_input.testcase_id - testcase_utils.emit_testcase_triage_duration_metric( - int(testcase_id), - testcase_utils.TESTCASE_TRIAGE_DURATION_REGRESSION_COMPLETED_STEP) - - if output.HasField('regression_task_output'): - task_output = output.regression_task_output - _update_build_metadata(output.uworker_input.job_type, - task_output.build_data_list) - _save_current_regression_range_indices(task_output, testcase_id) - if task_output.is_testcase_reproducible: - # Clear metadata from previous runs had it been marked as potentially - # flaky. - testcase = data_handler.get_testcase_by_id( - output.uworker_input.testcase_id) - task_creation.mark_unreproducible_if_flaky(testcase, 'regression', False) - - if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member - _ERROR_HANDLER.handle(output) - return - - save_regression_range(output) + # Retrieve the testcase early to be used by logs context. + testcase = data_handler.get_testcase_by_id(testcase_id) + with logs.regression_log_context(testcase, testcase.get_fuzz_target()): + testcase_utils.emit_testcase_triage_duration_metric( + int(testcase_id), + testcase_utils.TESTCASE_TRIAGE_DURATION_REGRESSION_COMPLETED_STEP) + + if output.HasField('regression_task_output'): + task_output = output.regression_task_output + _update_build_metadata(output.uworker_input.job_type, + task_output.build_data_list) + _save_current_regression_range_indices(task_output, testcase_id) + if task_output.is_testcase_reproducible: + # Clear metadata from previous runs had it been marked as potentially + # flaky. + testcase = data_handler.get_testcase_by_id( + output.uworker_input.testcase_id) + task_creation.mark_unreproducible_if_flaky(testcase, 'regression', + False) + + if output.error_type != uworker_msg_pb2.ErrorType.NO_ERROR: # pylint: disable=no-member + _ERROR_HANDLER.handle(output) + return + + save_regression_range(output) def _update_build_metadata( diff --git a/src/clusterfuzz/_internal/build_management/revisions.py b/src/clusterfuzz/_internal/build_management/revisions.py index e9de7f6803..aea659ed69 100644 --- a/src/clusterfuzz/_internal/build_management/revisions.py +++ b/src/clusterfuzz/_internal/build_management/revisions.py @@ -249,6 +249,9 @@ def _git_commit_position_to_git_hash_for_chromium(revision, repository): logs.error('Failed to parse git hash from url: ' + query_url) return None + sha = result_dict['git_sha'] + logs.info(f'git sha = {sha}') + return result_dict['git_sha'] @@ -370,7 +373,9 @@ def get_component_revisions_dict(revision, job_type, platform_id=None): if utils.is_chromium(): component = data_handler.get_component_name(job_type) repository = data_handler.get_repository_for_component(component) + logs.info('Is chrome, trying to figure out revision') if repository and not _is_clank(revision_vars_url_format): + logs.info('Hit codepath for chromium hash') revision_hash = _git_commit_position_to_git_hash_for_chromium( revision, repository) if revision_hash is None: @@ -389,6 +394,9 @@ def get_component_revisions_dict(revision, job_type, platform_id=None): # Use revision hash for info url later. revision = revision_hash + logs.info(f'revision hash = {revision_hash}, revisions_dict = {revisions_dict}') + else: + logs.info(f'DId not hit codepath for revision hash: repository={repository} and _is_clank={_is_clank(revision_vars_url_format)}') revision_vars_url = revision_vars_url_format % revision url_content = _get_url_content(revision_vars_url) diff --git a/src/clusterfuzz/_internal/cron/cleanup.py b/src/clusterfuzz/_internal/cron/cleanup.py index 41588ed7a0..aa3158bed3 100644 --- a/src/clusterfuzz/_internal/cron/cleanup.py +++ b/src/clusterfuzz/_internal/cron/cleanup.py @@ -38,6 +38,7 @@ from clusterfuzz._internal.metrics import crash_stats from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.metrics import monitoring_metrics +from clusterfuzz._internal.config import local_config GENERIC_INCORRECT_COMMENT = ( '\n\nIf this is incorrect, please add the {label_text}') @@ -1144,6 +1145,9 @@ def _sanitize_ccs_list(ccs_list): def update_issue_ccs_from_owners_file(policy, testcase, issue): """Add cc to an issue based on owners list from owners file. This is currently applicable to fuzz targets only.""" + if local_config.ProjectConfig().get('staging.enabled', False): + return + auto_cc_label = policy.label('auto_cc_from_owners') if not auto_cc_label: return @@ -1250,6 +1254,8 @@ def update_issue_owner_and_ccs_from_predator_results(policy, only_allow_ccs=False): """Assign the issue to an appropriate owner if possible.""" logs.info(f'{update_issue_owner_and_ccs_from_predator_results}') + if local_config.ProjectConfig().get('staging.enabled', False): + return if not issue or not issue.is_open: return diff --git a/src/clusterfuzz/_internal/cron/triage.py b/src/clusterfuzz/_internal/cron/triage.py index f7d10b3c77..e2b84e678f 100644 --- a/src/clusterfuzz/_internal/cron/triage.py +++ b/src/clusterfuzz/_internal/cron/triage.py @@ -353,7 +353,7 @@ def _file_issue(testcase, issue_tracker, throttler): file_exception = e if file_exception: - logs.error(f'Failed to file issue for testcase {testcase.key.id()}.') + logs.error(f'Failed to file issue for testcase {testcase.key.id()}: {file_exception}') monitoring_metrics.ISSUE_FILING.increment({ 'fuzzer_name': testcase.fuzzer_name, 'status': 'failed', diff --git a/src/clusterfuzz/_internal/google_cloud_utils/storage.py b/src/clusterfuzz/_internal/google_cloud_utils/storage.py index 531f6a8bb0..fdd6f70a7a 100644 --- a/src/clusterfuzz/_internal/google_cloud_utils/storage.py +++ b/src/clusterfuzz/_internal/google_cloud_utils/storage.py @@ -64,7 +64,7 @@ AUTH_TOKEN_EXPIRY_TIME = 10 * 60 # The number of retries to perform some GCS operation. -DEFAULT_FAIL_RETRIES = 8 +DEFAULT_FAIL_RETRIES = 0 # The time to wait between retries while performing GCS operation. DEFAULT_FAIL_WAIT = 2 diff --git a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py index 5538966d1e..3785281d51 100644 --- a/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py +++ b/src/clusterfuzz/_internal/issue_management/google_issue_tracker/issue_tracker.py @@ -748,6 +748,11 @@ def _update_issue(self, new_comment=None, notify=True): def save(self, new_comment=None, notify=True, skip_emails=[]): # pylint: disable=dangerous-default-value """Saves the issue.""" logs.info(f'Skipping supposed non gaia emails emails: {skip_emails}.') + # Just for a proof of concept + # Assert in runtime that we actually do not notify anyone + assert not self.assignee + assert not list(self._ccs) # This is a LabelStore + assert not list(self._collaborators) # Ditto if self._is_new: logs.info('google_issue_tracker: Creating new issue..') priority = _extract_label(self.labels, 'Pri-') or _DEFAULT_PRIORITY diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index 9b48d1963b..f60fdd42bc 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -13,16 +13,27 @@ # limitations under the License. """Logging functions.""" +import contextlib import datetime +import enum +import functools import json import logging from logging import config import os import socket import sys +import threading import time import traceback from typing import Any +from typing import NamedTuple +from typing import TYPE_CHECKING + +# This is needed to avoid circular import +if TYPE_CHECKING: + from clusterfuzz._internal.datastore.data_types import FuzzTarget + from clusterfuzz._internal.datastore.data_types import Testcase STACKDRIVER_LOG_MESSAGE_LIMIT = 80000 # Allowed log entry size is 100 KB. LOCAL_LOG_MESSAGE_LIMIT = 100000 @@ -197,8 +208,10 @@ def format(self, record: logging.LogRecord) -> str: record.name, 'pid': os.getpid(), - 'task_id': - os.getenv('CF_TASK_ID', 'null'), + 'release': + os.getenv('CLUSTERFUZZ_RELEASE', 'prod'), + 'docker_image': + os.getenv('DOCKER_IMAGE', '') } initial_payload = os.getenv('INITIAL_TASK_PAYLOAD') @@ -492,6 +505,25 @@ def _add_appengine_trace(extras): project_id=project_id, trace_id=trace_id) +def intercept_log_context(func): + """Intercepts the wrapped function and injects metadata + into the kwargs for a given log context + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not kwargs.get('ignore_context'): + for context in log_contexts.contexts: + kwargs.update(context.get_extras()._asdict()) + else: + # This is needed to avoid logging the label 'ingore_context: True' + del kwargs["ignore_context"] + return func(*args, **kwargs) + + return wrapper + + +@intercept_log_context def emit(level, message, exc_info=None, **extras): """Log in JSON.""" logger = get_logger() @@ -538,9 +570,7 @@ def emit(level, message, exc_info=None, **extras): 'path': path_name, 'line': line_number, 'method': method_name - }, - 'release': os.environ.get('CLUSTERFUZZ_RELEASE', 'prod'), - 'docker_image': os.environ.get('DOCKER_IMAGE', '') + } }) @@ -576,3 +606,208 @@ def log_fatal_and_exit(message, **extras): info('Waiting for %d seconds before exit.' % wait_before_exit) time.sleep(wait_before_exit) sys.exit(-1) + + +class GenericLogStruct(NamedTuple): + pass + + +class TaskLogStruct(NamedTuple): + task_id: str + task_name: str + task_argument: str + task_job_name: str + stage: str + + +class TestcaseLogStruct(NamedTuple): + testcase_id: str + fuzz_target: str + job: str + fuzzer: str + + +class LogContextType(enum.Enum): + """Log context types + This is the way to define the context for a given entrypoint + and this context is used for define the adicional labels + to be added to the log. + """ + TASK = 'task' + TESTCASE = 'testcase' + PROGRESSION = 'progression' + REGRESSION = 'regression' + + def get_extras(self) -> NamedTuple: + """Get the structured log for a given context""" + if self == LogContextType.TASK: + stage = log_contexts.meta.get('stage', Stage.UNKNOWN).value + try: + task_id = os.getenv('CF_TASK_ID', 'null') + task_name = os.getenv('CF_TASK_NAME', 'null') + task_argument = os.getenv('CF_TASK_ARGUMENT', 'null') + task_job_name = os.getenv('CF_TASK_JOB_NAME', 'null') + return TaskLogStruct( + task_id=task_id, + task_name=task_name, + task_argument=task_argument, + stage=stage, + task_job_name=task_job_name) + except: + # This flag is necessary to avoid + # infinite loop in this context verification. + error('Error retrieving context for task logs.', ignore_context=True) + return GenericLogStruct() + + elif self == LogContextType.TESTCASE: + try: + testcase: 'Testcase | None' = log_contexts.meta.get('testcase') + if not testcase: + error( + 'Testcase not found in log context metadata.', + ignore_context=True) + return GenericLogStruct() + + fuzz_target: 'FuzzTarget | None' = log_contexts.meta.get('fuzz_target') + if fuzz_target and fuzz_target.binary: + fuzz_target_bin = fuzz_target.binary + else: + fuzz_target_bin = testcase.get_metadata('fuzzer_binary_name', + 'unknown') + + return TestcaseLogStruct( + testcase_id=testcase.key.id(), # type: ignore + fuzz_target=fuzz_target_bin, # type: ignore + job=testcase.job_type, # type: ignore + fuzzer=testcase.fuzzer_name # type: ignore + ) + except: + error( + 'Error retrieving context for testcase-based logs.', + ignore_context=True) + return GenericLogStruct() + + elif self == LogContextType.PROGRESSION: + # Field to add specific metadata for progression. + return GenericLogStruct() + + elif self == LogContextType.REGRESSION: + # Field to add specific metadata for regression. + return GenericLogStruct() + + return GenericLogStruct() + + +class Stage(enum.Enum): + PREPROCESS = 'preprocess' + MAIN = 'main' + POSTPROCESS = 'postprocess' + UNKNOWN = 'unknown' + + +class Singleton(type): + _instances = {} + _lock = threading.Lock() + + def __call__(cls, *args, **kwargs): + with cls._lock: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + +class LogContexts(metaclass=Singleton): + """Class to keep the log contexts and metadata""" + + def __init__(self): + self.contexts: list[LogContextType] = [] + self.meta: dict[Any, Any] = {} + self._data_lock = threading.Lock() + + def add(self, new_contexts: list[LogContextType]): + with self._data_lock: + self.contexts += new_contexts + + def add_metadata(self, key: Any, value: Any): + with self._data_lock: + self.meta[key] = value + + def delete(self, contexts: list[LogContextType]): + with self._data_lock: + for ctx in contexts: + self.contexts.remove(ctx) + + def delete_metadata(self, key: Any): + with self._data_lock: + if key in self.meta: + del self.meta[key] + + def clear(self): + with self._data_lock: + self.contexts = [] + + +log_contexts = LogContexts() + + +@contextlib.contextmanager +def wrap_log_context(contexts: list[LogContextType]): + try: + log_contexts.add(contexts) + yield + finally: + log_contexts.delete(contexts) + + +@contextlib.contextmanager +def task_stage_context(stage: Stage): + """Creates a task context for a given stage""" + with wrap_log_context(contexts=[LogContextType.TASK]): + try: + log_contexts.add_metadata('stage', stage) + yield + except Exception as e: + error(message='Error during task.') + raise e + finally: + log_contexts.delete_metadata('stage') + + +@contextlib.contextmanager +def testcase_log_context(testcase: 'Testcase', + fuzz_target: 'FuzzTarget | None'): + """Creates a testcase-based context for a given testcase. + + Fuzz target as an argument is needed since retrieving this entity depends on + the task's stage. In trusted part, it can be retrieved by querying the DB, + while in untrusted part is only accessible through the protobuf. + """ + with wrap_log_context(contexts=[LogContextType.TESTCASE]): + try: + log_contexts.add_metadata('testcase', testcase) + log_contexts.add_metadata('fuzz_target', fuzz_target) + yield + except Exception as e: + error(message='Error during testcase context.') + raise e + finally: + log_contexts.delete_metadata('testcase') + log_contexts.delete_metadata('fuzz_target') + + +# Keeping a context for each testcase-based task to make it +# easier to add speficic metadata if needed. +@contextlib.contextmanager +def progression_log_context(testcase: 'Testcase', + fuzz_target: 'FuzzTarget | None'): + with testcase_log_context(testcase, fuzz_target): + with wrap_log_context(contexts=[LogContextType.PROGRESSION]): + yield + + +@contextlib.contextmanager +def regression_log_context(testcase: 'Testcase', + fuzz_target: 'FuzzTarget | None'): + with testcase_log_context(testcase, fuzz_target): + with wrap_log_context(contexts=[LogContextType.REGRESSION]): + yield diff --git a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py index d8cab4021a..17d91a6eea 100644 --- a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py +++ b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py @@ -23,6 +23,7 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.tests.test_libs import helpers +from clusterfuzz._internal.tests.test_libs import test_utils class GetSourceLocationTest(unittest.TestCase): @@ -171,13 +172,14 @@ def test_format_record(self): self.assertEqual({ 'message': 'log message', 'created': '1970-01-01T00:00:10Z', + 'docker_image': '', 'severity': 'INFO', 'bot_name': 'linux-bot', 'task_payload': 'fuzz fuzzer1 job1', 'fuzz_target': 'fuzz_target1', 'name': 'logger_name', 'pid': 1337, - 'task_id': 'job-1337', + 'release': 'prod', 'extras': { 'a': 1, }, @@ -198,12 +200,13 @@ def test_no_extras(self): self.assertEqual({ 'message': 'log message', 'created': '1970-01-01T00:00:10Z', + 'docker_image': '', 'severity': 'INFO', 'bot_name': 'linux-bot', 'task_payload': 'fuzz fuzzer1 job1', 'name': 'logger_name', - 'task_id': 'job-1337', 'pid': 1337, + 'release': 'prod', 'location': { 'path': 'path', 'line': 123, @@ -220,6 +223,7 @@ def test_worker_bot_name(self): record.extras = None self.assertEqual({ + 'docker_image': '', 'message': 'log message', 'created': '1970-01-01T00:00:10Z', 'severity': 'INFO', @@ -227,8 +231,8 @@ def test_worker_bot_name(self): 'worker_bot_name': 'worker', 'task_payload': 'fuzz fuzzer1 job1', 'name': 'logger_name', - 'task_id': 'job-1337', 'pid': 1337, + 'release': 'prod', 'location': { 'path': 'path', 'line': 123, @@ -277,7 +281,6 @@ def test_format_basic(self): self.assertEqual(json_result['severity'], 'INFO') self.assertEqual(json_result['name'], 'test_logger') self.assertEqual(json_result['pid'], os.getpid()) - self.assertEqual(json_result['task_id'], 'null') self.assertTrue('location' in json_result) self.assertFalse('extras' in json_result) @@ -292,7 +295,6 @@ def test_format_with_env_vars(self): self.assertEqual(json_result['bot_name'], 'test_bot') self.assertEqual(json_result['task_payload'], 'test_payload') - self.assertEqual(json_result['task_id'], '123') def test_format_with_initial_payload(self): """Tests formatting with initial task payload.""" @@ -394,18 +396,31 @@ def test_configure_appengine(self): self.assertEqual(0, self.mock.dictConfig.call_count) +@test_utils.with_cloud_emulators('datastore') class EmitTest(unittest.TestCase): """Test emit.""" def setUp(self): helpers.patch(self, [ 'clusterfuzz._internal.metrics.logs.get_logger', - 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine' + 'clusterfuzz._internal.metrics.logs._is_running_on_app_engine', + 'clusterfuzz._internal.datastore.data_types.Testcase.get_fuzz_target', ]) + os.environ['CF_TASK_ID'] = 'f61826c3-ca9a-4b97-9c1e-9e6f4e4f8868' + os.environ['CF_TASK_NAME'] = 'fuzz' + os.environ['CF_TASK_ARGUMENT'] = 'libFuzzer' + os.environ['CF_TASK_JOB_NAME'] = 'libfuzzer_asan_gopacket' # Reset default extras as it may be modified during other test runs. logs._default_extras = {} # pylint: disable=protected-access self.mock._is_running_on_app_engine.return_value = False # pylint: disable=protected-access + def tearDown(self): + del os.environ['CF_TASK_ID'] + del os.environ['CF_TASK_NAME'] + del os.environ['CF_TASK_ARGUMENT'] + del os.environ['CF_TASK_JOB_NAME'] + return super().tearDown() + def test_no_logger(self): """Test no logger.""" self.mock.get_logger.return_value = None @@ -428,8 +443,6 @@ def test_info(self): 'target': 'bot', 'test': 'yes' }, - 'release': 'prod', - 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, @@ -454,8 +467,6 @@ def test_error(self): 'target': 'bot', 'test': 'yes' }, - 'release': 'prod', - 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, @@ -463,6 +474,194 @@ def test_error(self): } }) + @logs.task_stage_context(logs.Stage.PREPROCESS) + def test_task_log_context(self): + """Test that the logger is called with the + correct arguments considering the log context and metadata + """ + logger = mock.MagicMock() + self.mock.get_logger.return_value = logger + self.assertEqual(logs.log_contexts.contexts, [logs.LogContextType.TASK]) + self.assertEqual(logs.log_contexts.meta, {'stage': logs.Stage.PREPROCESS}) + statement_line = inspect.currentframe().f_lineno + 1 + logs.emit(logging.ERROR, 'msg', exc_info='ex', target='bot', test='yes') + + logger.log.assert_called_once_with( + logging.ERROR, + 'msg', + exc_info='ex', + extra={ + 'extras': { + 'target': 'bot', + 'test': 'yes', + 'task_id': 'f61826c3-ca9a-4b97-9c1e-9e6f4e4f8868', + 'task_name': 'fuzz', + 'task_argument': 'libFuzzer', + 'task_job_name': 'libfuzzer_asan_gopacket', + 'stage': 'preprocess' + }, + 'location': { + 'path': os.path.abspath(__file__).rstrip('c'), + 'line': statement_line, + 'method': 'test_task_log_context' + }, + }) + + def test_progression_log_context(self): + """Test that the logger is called with the + correct arguments considering the log context and metadata + """ + from clusterfuzz._internal.datastore import data_types + logger = mock.MagicMock() + self.mock.get_logger.return_value = logger + fuzz_target = data_types.FuzzTarget( + id='libFuzzer_abc', engine='libFuzzer', binary='abc') + fuzz_target.put() + self.mock.get_fuzz_target.return_value = fuzz_target + testcase = data_types.Testcase( + fuzzer_name="test_fuzzer", job_type='test_job') + testcase.put() + + with logs.progression_log_context(testcase, fuzz_target): + self.assertEqual( + logs.log_contexts.contexts, + [logs.LogContextType.TESTCASE, logs.LogContextType.PROGRESSION]) + self.assertEqual(logs.log_contexts.meta, { + 'testcase': testcase, + 'fuzz_target': fuzz_target + }) + statement_line = inspect.currentframe().f_lineno + 1 + logs.emit(logging.ERROR, 'msg', exc_info='ex', target='bot', test='yes') + + logger.log.assert_called_with( + logging.ERROR, + 'msg', + exc_info='ex', + extra={ + 'extras': { + 'target': 'bot', + 'test': 'yes', + 'testcase_id': 1, + 'fuzz_target': 'abc', + 'job': 'test_job', + 'fuzzer': 'test_fuzzer' + }, + 'location': { + 'path': os.path.abspath(__file__).rstrip('c'), + 'line': statement_line, + 'method': 'test_progression_log_context' + }, + }) + + def test_task_context_catches_and_logs_exception(self): + """Checks that the task_stage_context catches and logs + the error raised in the decorated scope. + """ + logger = mock.MagicMock() + self.mock.get_logger.return_value = logger + + with logs.task_stage_context(logs.Stage.PREPROCESS): + try: + exception = Exception('msg') + raise exception + except Exception: + statement_line = inspect.currentframe().f_lineno + 1 + logs.error('xpto') + logger.log.assert_called_with( + logging.ERROR, + 'xpto', + exc_info=sys.exc_info(), + extra={ + 'extras': { + 'task_id': 'f61826c3-ca9a-4b97-9c1e-9e6f4e4f8868', + 'task_name': 'fuzz', + 'task_argument': 'libFuzzer', + 'task_job_name': 'libfuzzer_asan_gopacket', + 'stage': 'preprocess' + }, + 'location': { + 'path': os.path.abspath(__file__).rstrip('c'), + 'line': statement_line, + 'method': 'test_task_context_catches_and_logs_exception' + }, + }) + + @logs.task_stage_context(logs.Stage.PREPROCESS) + def test_log_ignore_context(self): + """Test that the emit interceptor ignores contect + when passed the ignore_context flag + """ + logger = mock.MagicMock() + self.mock.get_logger.return_value = logger + + self.assertEqual(logs.log_contexts.contexts, [logs.LogContextType.TASK]) + self.assertEqual(logs.log_contexts.meta, {'stage': logs.Stage.PREPROCESS}) + statement_line = inspect.currentframe().f_lineno + 1 + logs.emit( + logging.ERROR, + 'msg', + exc_info='ex', + target='bot', + test='yes', + ignore_context=True) + + logger.log.assert_called_once_with( + logging.ERROR, + 'msg', + exc_info='ex', + extra={ + 'extras': { + 'target': 'bot', + 'test': 'yes', + }, + 'location': { + 'path': os.path.abspath(__file__).rstrip('c'), + 'line': statement_line, + 'method': 'test_log_ignore_context' + }, + }) + + def test_missing_fuzz_target_in_log_context(self): + """Test the testcase-based log context when the fuzz target is missing.""" + from clusterfuzz._internal.datastore import data_types + logger = mock.MagicMock() + self.mock.get_logger.return_value = logger + testcase = data_types.Testcase( + fuzzer_name="test_fuzzer", job_type='test_job') + testcase.set_metadata('fuzzer_binary_name', 'fuzz_abc') + testcase.put() + + with logs.regression_log_context(testcase, None): + self.assertEqual( + logs.log_contexts.contexts, + [logs.LogContextType.TESTCASE, logs.LogContextType.REGRESSION]) + self.assertEqual(logs.log_contexts.meta, { + 'testcase': testcase, + 'fuzz_target': None + }) + statement_line = inspect.currentframe().f_lineno + 1 + logs.emit(logging.ERROR, 'msg', exc_info='ex', target='bot', test='yes') + + logger.log.assert_called_with( + logging.ERROR, + 'msg', + exc_info='ex', + extra={ + 'extras': { + 'target': 'bot', + 'test': 'yes', + 'testcase_id': 1, + 'fuzz_target': 'fuzz_abc', + 'job': 'test_job', + 'fuzzer': 'test_fuzzer' + }, + 'location': { + 'path': os.path.abspath(__file__).rstrip('c'), + 'line': statement_line, + 'method': 'test_missing_fuzz_target_in_log_context' + }, + }) + class TruncateTest(unittest.TestCase): """Test truncate.""" @@ -497,3 +696,44 @@ def test_exception(self): logs.error('test', exception='exception', hello='1') self.mock.emit.assert_called_once_with( logging.ERROR, 'test', exc_info='err', hello='1') + + +class TestLogContextSingleton(unittest.TestCase): + """Tests for the log context singleton + It checks the singleton behavior works and is thread safe + """ + + def test_is_same(self): + """Test the singleton is the same instance for different module loads""" + from clusterfuzz._internal.base.tasks.task_rate_limiting import \ + logs as logs_from_task_rate_limiting + from python.bot.startup.run_bot import logs as logs_from_run_bot + + self.assertIs(logs_from_run_bot, logs_from_task_rate_limiting) + self.assertIs(logs_from_run_bot.log_contexts, + logs_from_run_bot.LogContexts()) + logs_from_run_bot.log_contexts.add([logs_from_run_bot.LogContextType.TASK]) + + self.assertEqual(logs_from_task_rate_limiting.log_contexts, + logs_from_run_bot.log_contexts) + logs_from_run_bot.log_contexts.clear() + + def test_multi_threading(self): + """Test multithread""" + + def incrementer(): + from python.bot.startup.run_bot import logs as run_bot_logs + run_bot_logs.log_contexts.add([logs.LogContextType.TASK]) + + import threading + threads = [] + for _ in range(5): + thread = threading.Thread(target=incrementer) + threads.append(thread) + thread.start() + + for thread in threads: + thread.join() + + from python.bot.startup.run_bot import logs as run_bot_logs + self.assertEqual(len(run_bot_logs.log_contexts.contexts), 5) diff --git a/src/local/butler/appengine.py b/src/local/butler/appengine.py index 87a59433d4..982b578b07 100644 --- a/src/local/butler/appengine.py +++ b/src/local/butler/appengine.py @@ -108,7 +108,7 @@ def symlink_dirs(): def build_templates(): """Build template files used in appengine.""" - common.execute('python polymer_bundler.py', cwd='local') + common.execute('python3 polymer_bundler.py', cwd='local') def symlink_config_dir(): diff --git a/src/local/butler/common.py b/src/local/butler/common.py index 72f50ec68f..50d07a2f10 100644 --- a/src/local/butler/common.py +++ b/src/local/butler/common.py @@ -249,7 +249,7 @@ def _pipfile_to_requirements(pipfile_dir, requirements_path, dev=False): dev_arg = '--dev' return_code, output = execute( - f'python -m pipenv requirements {dev_arg}', + f'python3.11 -m pipenv requirements {dev_arg}', exit_on_error=False, cwd=pipfile_dir, extra_environments={'PIPENV_IGNORE_VIRTUALENVS': '1'}, @@ -257,7 +257,7 @@ def _pipfile_to_requirements(pipfile_dir, requirements_path, dev=False): if return_code != 0: # Older pipenv version. return_code, output = execute( - f'python -m pipenv lock -r --no-header {dev_arg}', + f'python3.11 -m pipenv lock -r --no-header {dev_arg}', exit_on_error=False, cwd=pipfile_dir, extra_environments={'PIPENV_IGNORE_VIRTUALENVS': '1'}, @@ -418,7 +418,9 @@ def kill_leftover_emulators(): kill_process('dev_appserver.py') kill_process('CloudDatastore.jar') kill_process('pubsub-emulator') - kill_process('run_bot') + # It was needed to add the "buttler.py" because I was killing + # the testing process for running the "run_bot_test" + kill_process('butler.py run_bot') def get_platform(): diff --git a/src/local/butler/create_config.py b/src/local/butler/create_config.py index 4518bba4a5..1289b17acd 100755 --- a/src/local/butler/create_config.py +++ b/src/local/butler/create_config.py @@ -53,6 +53,7 @@ 'replicapool.googleapis.com', 'replicapoolupdater.googleapis.com', 'resourceviews.googleapis.com', + 'secretmanager.googleapis.com', 'siteverification.googleapis.com', 'sourcerepo.googleapis.com', 'stackdriver.googleapis.com', @@ -72,7 +73,7 @@ def __init__(self, oauth_client_secrets_path): flow = InstalledAppFlow.from_client_secrets_file( oauth_client_secrets_path, scopes=['https://www.googleapis.com/auth/siteverification']) - credentials = flow.run_local_server() + credentials = flow.run_local_server(open_browser=False) http = google_auth_httplib2.AuthorizedHttp( credentials, http=httplib2.Http()) @@ -197,7 +198,7 @@ def deploy_appengine(gcloud, config_dir, appengine_location): gcloud.run('app', 'create', '--region=' + appengine_location) subprocess.check_call([ - 'python', 'butler.py', 'deploy', '--force', '--targets', 'appengine', + 'python3', 'butler.py', 'deploy', '--force', '--targets', 'appengine', '--prod', '--config-dir', config_dir ]) diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 8582a85ca5..73549744cd 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -348,28 +348,6 @@ def _update_bigquery(project): 'datasets.yaml')) -def _update_redis(project): - """Update redis instance.""" - _update_deployment_manager( - project, 'redis', - os.path.join(environment.get_config_directory(), 'redis', - 'instance.yaml')) - - region = appengine.region(project) - return_code, _ = common.execute( - 'gcloud compute networks vpc-access connectors describe ' - 'connector --region={region} ' - '--project={project}'.format(project=project, region=region), - exit_on_error=False) - - if return_code: - # Does not exist. - common.execute('gcloud compute networks vpc-access connectors create ' - 'connector --network=default --region={region} ' - '--range=10.8.0.0/28 ' - '--project={project}'.format(project=project, region=region)) - - def get_remote_sha(): """Get remote sha of origin/master.""" _, remote_sha_line = common.execute('git ls-remote origin refs/heads/master') @@ -432,7 +410,6 @@ def _prod_deployment_helper(config_dir, _update_pubsub_queues(project) _update_alerts(project) _update_bigquery(project) - _update_redis(project) labels: dict[str, Any] = { 'deploy_zip': bool(package_zip_paths), @@ -444,6 +421,12 @@ def _prod_deployment_helper(config_dir, } try: + # AppEngine depends on the REDIS_HOST env var being available + # Therefore, terraform must run first, since it creates redis + if deploy_k8s: + _deploy_terraform(config_dir) + _deploy_k8s(config_dir) + _deploy_app_prod( project, deployment_bucket, @@ -457,9 +440,7 @@ def _prod_deployment_helper(config_dir, common.execute( f'python butler.py run setup --config-dir {config_dir} --non-dry-run') - if deploy_k8s: - _deploy_terraform(config_dir) - _deploy_k8s(config_dir) + print(f'Production deployment finished. {labels}') monitoring_metrics.PRODUCTION_DEPLOYMENT.increment(labels) diff --git a/src/local/butler/scripts/migrate_blobs.py b/src/local/butler/scripts/migrate_blobs.py new file mode 100644 index 0000000000..dbe670b1e7 --- /dev/null +++ b/src/local/butler/scripts/migrate_blobs.py @@ -0,0 +1,180 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Executes update task locally, so we can run it through a debugger.""" + +from clusterfuzz._internal.system import environment +from clusterfuzz._internal.datastore import data_types +from clusterfuzz._internal.google_cloud_utils import storage +from clusterfuzz._internal.google_cloud_utils import credentials +from clusterfuzz._internal.google_cloud_utils import blobs +from local.butler import common +from google.cloud import ndb +from clusterfuzz._internal.base import utils + +import contextlib +import time + +BACKOFF_BASE_INTERVAL = 2 +MAX_RETRIES = 3 + + +#TODO(vitorguidi): generalize this so we can point to other projects +prod_blob_bucket = 'clusterfuzz-blobs' +staging_blob_bucket = 'blobs.clusterfuzz-staging.appspot.com' +prod_bucket_domains = [ + 'cluster-fuzz.appspot.com', + 'clusterfuzz.com', +] +staging_bucket_domain = 'clusterfuzz-staging.appspot.com' + +def _copy_blob(origin_blob_path, target_blob_path): + assert not prod_blob_bucket in target_blob_path + assert staging_blob_bucket in target_blob_path + + storage.copy_blob(origin_blob_path, target_blob_path) + + storage.copy_blob(origin_blob_path, target_blob_path ) + print(f'Copied blob {origin_blob_path} to {target_blob_path}') + +def _migrate_gcs_blob(source_blob_key): + print(f'Moving the {source_blob_key} blob in prod to the same name in the staging bucket.') + origin_blob_path = f'gs://{prod_blob_bucket}/{source_blob_key}' + target_blob_path = f'gs://{staging_blob_bucket}/{source_blob_key}' + try: + _copy_blob(origin_blob_path, target_blob_path) + except Exception as e: + # This might have be a legacy blobkey in prod which + # was migrated to a gcs ke in staging. This should be + # idempotent + origin_blob_path = target_blob_path + _copy_blob(origin_blob_path, target_blob_path) + finally: + # The blob key remains the same before and after migration + return source_blob_key + + +def _migrate_legacy_blob(source_blob_key): + legacy_blob_info = ndb.Key(blobs._blobmigrator_BlobKeyMapping, source_blob_key).get() + source_blob_path = legacy_blob_info.gcs_filename + new_blob_key = blobs.generate_new_blob_name() + + _copy_blob(f'gs:/{source_blob_path}', f'gs://{staging_blob_bucket}/{new_blob_key}') + return new_blob_key + +def migrate_blob(source_blob_key): + ''' + Takes the source blob from the production bucket and copies it to + a target blob in the staging bucket. This is safe to run multiple times, + since either: + - The source blob is a gcs url, and will be copied with the same + blob key over and over, so this is idempotent; + - Or the source blob is a legacy key, mapped to a gcs object in prod, + with the mapping in the _blogmigrator_BlobKeyMapping entity. We keep the blob key, + and only the gcs_filename changes. We copy the same file everytime, so idempotency ensues. + ''' + + if blobs._is_gcs_key(source_blob_key): + return _migrate_gcs_blob(source_blob_key) + else: + return _migrate_legacy_blob(source_blob_key) + +def migrate_bucket(source_bucket, target_bucket): + ''' + Creates the target bucket, if it does not exist, and + moves all content from the source bucket into it. + ''' + + bucket_exists = storage.create_bucket_if_needed(target_bucket) + + assert bucket_exists + + error_code, output = common.execute(f'gcloud storage rsync --recursive gs://{source_bucket} gs://{target_bucket}', exit_on_error=False) + output_as_str = str(output) + print(output_as_str) + if error_code == 0: + print(f'Migrated bucket contents from {source_bucket} to {target_bucket}') + else: + print(f'Failed to migrate bucket: {output_as_str}') + +def migrate_data_bundle(data_bundle): + ''' + Migrates a data bundle from prod to staging by replicating the respective blob from + blobstore_key, keeping the same key, and moving the bucket contents from the production + bucket to the corresponding one in staging. + ''' + print(f'Migrating data bundle {data_bundle.name}') + bundle_corpus_gcs_bucket = data_bundle.bucket_name + if '__common' in data_bundle.name: + print(f'Data bundle {data_bundle.name} is deprecated, skipping') + return + target_corpus_bucket = '' + for domain in prod_bucket_domains: + if domain in bundle_corpus_gcs_bucket: + target_corpus_bucket = bundle_corpus_gcs_bucket.replace(domain, staging_bucket_domain) + + assert staging_bucket_domain in target_corpus_bucket + + migrate_bucket(bundle_corpus_gcs_bucket, target_corpus_bucket) + data_bundle.bucket_name = target_corpus_bucket + data_bundle.put() + +def migrate_fuzzer(fuzzer): + ''' + Migrates a fuzzer from production to staging. It suffices to replicate the + blobstore key to the staging blobs bucket, keeping the same key. + ''' + print(f'Migrating fuzzer {fuzzer.name}') + source_blob = fuzzer.blobstore_key + if not source_blob: + print('No blobstore_key, skipping') + return + new_blob_key = migrate_blob(source_blob) + fuzzer.blobstore_key = new_blob_key + fuzzer.put() + print(f'Migrated fuzzer {fuzzer.name} to gcs://{staging_blob_bucket}/{new_blob_key}') + +def migrate_job(job): + ''' + Migrates a job from production to staging. It suffices to replicate the + custom binary blob to the staging blobs bucket, keeping the same key. + ''' + source_blob = job.custom_binary_key + if not source_blob: + print(f'No custom binary key, skipping') + return + new_blob_key = migrate_blob(source_blob) + job.custom_binary_key = new_blob_key + job.put() + print(f'Migrated job {job.name} to gcs://{staging_blob_bucket}/{new_blob_key}') + + +def execute(args): #pylint: disable=unused-argument + """Build keywords.""" + environment.set_bot_environment() + # print('checking jobs') + # for job in data_types.Job.query(): + # migrate_job(job) + # print('\n\n') + + # print('checking fuzzers') + # for fuzzer in data_types.Fuzzer.query(): + # migrate_fuzzer(fuzzer) + # print('\n\n') + + print('checking databundle') + for data_bundle in data_types.DataBundle.query(): + migrate_data_bundle(data_bundle) + print('\n\n') + + print('done') diff --git a/src/local/butler/scripts/run_task.py b/src/local/butler/scripts/run_task.py new file mode 100644 index 0000000000..b079cb5063 --- /dev/null +++ b/src/local/butler/scripts/run_task.py @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run a task locally.""" + +from clusterfuzz._internal.bot.fuzzers import init +from clusterfuzz._internal.bot.tasks import commands +from clusterfuzz._internal.system import environment + +# from local.butler.run_bot + + +def execute(args): + """Build keywords.""" + environment.set_bot_environment() + init.run() + commands.process_command_impl('fuzz', 'lokihardt_jshitter', 'linux32_asan_d8_dbg', + True, True) + pass + + \ No newline at end of file