[SVCS-552] folder_file_op cut down revalidate calls.

tests/core/fixtures.py

@@ -0,0 +1,31 @@
+import pytest
+
+from tests import utils
+from waterbutler.core.path import WaterButlerPath
+
+
+@pytest.fixture
+def provider1():
+    return utils.MockProvider1({'user': 'name'}, {'pass': 'word'}, {})
+
+
+@pytest.fixture
+def provider2():
+    return utils.MockProvider2({'user': 'name'}, {'pass': 'phrase'}, {})
+
+
+@pytest.fixture
+def dest_path():
+    return WaterButlerPath('/older folder/new folder/')
+
+@pytest.fixture
+def folder_children():
+    return [utils.MockFileMetadata(), utils.MockFolderMetadata(), utils.MockFileMetadata()]
+
+@pytest.fixture
+def src_path():
+    return WaterButlerPath('/folder with children/')
+
+@pytest.fixture
+def mock_folder():
+    return utils.MockFolderMetadata()

tests/core/test_provider.py

@@ -5,16 +5,14 @@
 from waterbutler.core import metadata
 from waterbutler.core import exceptions
 
-
-@pytest.fixture
-def provider1():
-    return utils.MockProvider1({'user': 'name'}, {'pass': 'word'}, {})
-
-
-@pytest.fixture
-def provider2():
-    return utils.MockProvider2({'user': 'name'}, {'pass': 'phrase'}, {})
-
+from tests.core.fixtures import (
+    provider1,
+    provider2,
+    src_path,
+    dest_path,
+    folder_children,
+    mock_folder
+)
 
 class TestBaseProvider:
 
@@ -481,3 +479,76 @@ def test_build_range_header(self, provider1):
         assert 'bytes=10-' == provider1._build_range_header((10, None))
         assert 'bytes=10-100' == provider1._build_range_header((10, 100))
         assert 'bytes=-255' == provider1._build_range_header((None, 255))
+
+
+class TestFolderFileOp:
+
+    @pytest.mark.asyncio
+    async def test_folder_file_op_overwrite(self,
+                                            provider1,
+                                            provider2,
+                                            src_path,
+                                            dest_path,
+                                            mock_folder,
+                                            folder_children):
+        provider2.create_folder = utils.MockCoroutine(return_value=mock_folder)
+        provider1.metadata = utils.MockCoroutine(return_value=folder_children)
+        provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True))
+
+        folder, created = await provider1._folder_file_op(provider1.copy,
+                                                          provider2,
+                                                          src_path,
+                                                          dest_path)
+
+        provider2.create_folder.assert_called_with(dest_path, folder_precheck=False)
+        provider1.metadata.assert_called_with(src_path)
+
+        assert (folder, created) == (mock_folder, False)
+
+        # The order of these lists does not matter, but they should contain all the same elements.
+        assert all(child for child in folder.children if child in folder_children)
+        assert all(child for child in folder_children if child in folder.children)
+
+    @pytest.mark.asyncio
+    async def test_folder_file_op_new(self,
+                                      provider1,
+                                      provider2,
+                                      src_path,
+                                      dest_path,
+                                      mock_folder,
+                                      folder_children):
+        provider2.create_folder = utils.MockCoroutine(return_value=mock_folder)
+        provider2.delete = utils.MockCoroutine(side_effect=exceptions.ProviderError('test message',
+                                                                                    code=404))
+        provider1.metadata = utils.MockCoroutine(return_value=folder_children)
+        provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True))
+
+        folder, created = await provider1._folder_file_op(provider1.copy,
+                                                          provider2,
+                                                          src_path,
+                                                          dest_path)
+        provider2.create_folder.assert_called_with(dest_path, folder_precheck=False)
+        provider1.metadata.assert_called_with(src_path)
+
+        assert (folder, created) == (mock_folder, True)
+
+        # The order of these lists does not matter, but they should contain all the same elements.
+        assert all(child for child in folder.children if child in folder_children)
+        assert all(child for child in folder_children if child in folder.children)
+
+    @pytest.mark.asyncio
+    async def test_folder_file_op_raises(self,
+                                         provider1,
+                                         provider2,
+                                         src_path,
+                                         dest_path,
+                                         mock_folder,
+                                         folder_children):
+        provider2.create_folder = utils.MockCoroutine(return_value=mock_folder)
+        provider2.delete = utils.MockCoroutine(side_effect=exceptions.ProviderError('test message',
+                                                                                    code=500))
+        provider1.metadata = utils.MockCoroutine(return_value=folder_children)
+        provider1.copy = utils.MockCoroutine(return_value=(utils.MockFolderMetadata(), True))
+
+        with pytest.raises(exceptions.ProviderError):
+            await provider1._folder_file_op(provider1.copy, provider2, src_path, dest_path)

tests/providers/bitbucket/test_metadata.py

@@ -27,6 +27,7 @@ def test_build_file_metadata(self, file_metadata, owner, repo):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == full_path
         assert metadata.name == name
         assert metadata.path == full_path
         assert metadata.kind == 'file'

tests/providers/box/test_metadata.py

@@ -16,6 +16,7 @@ def test_file_metadata(self, root_provider_fixtures):
         item = root_provider_fixtures['file_metadata']['entries'][0]
         dest_path = WaterButlerPath('/charmander/name.txt', _ids=('0', item['id'], item['id']))
         data = BoxFileMetadata(item, dest_path)
+        assert data.id == '5000948880'
         assert data.name == 'tigers.jpeg'
         assert data.path == '/5000948880'
         assert data.provider == 'box'

tests/providers/cloudfiles/test_metadata.py

@@ -39,6 +39,7 @@ def test_header_metadata(self, file_header_metadata_txt):
 
         path = WaterButlerPath('/file.txt')
         data = CloudFilesHeaderMetadata(file_header_metadata_txt, path.path)
+        assert data.id == '/file.txt'
         assert data.name == 'file.txt'
         assert data.path == '/file.txt'
         assert data.provider == 'cloudfiles'

tests/providers/dataverse/test_metadata.py

@@ -32,6 +32,7 @@ def test_revision_metadata(self, revision_metadata_object):
 class TestFileMetadata:
 
     def test_file_metadata(self, file_metadata_object):
+        assert file_metadata_object.id == '20'
         assert file_metadata_object.is_file
         assert not file_metadata_object.is_folder
         assert file_metadata_object.provider == 'dataverse'

tests/providers/dropbox/test_metadata.py

@@ -12,6 +12,7 @@ class TestDropboxMetadata:
     def test_file_metadata(self, provider_fixtures):
         data = DropboxFileMetadata(provider_fixtures['file_metadata'], '/Photos')
 
+        assert data.id == 'id:8y8sAJlrhuAAAAAAAAAAAQ'
         assert data.name == 'Getting_Started.pdf'
         assert data.path == '/Getting_Started.pdf'
         assert data.size == 124778
@@ -88,6 +89,7 @@ def test_file_metadata(self, provider_fixtures):
     def test_folder_metadata(self, provider_fixtures):
         data = DropboxFolderMetadata(provider_fixtures['folder_metadata'], '/Photos')
 
+        assert data.id == 'id:67BLXqRKo-gAAAAAAAADZg'
         assert data.name == 'newfolder'
         assert data.path == '/newfolder/'
         assert data.etag is None

tests/providers/filesystem/test_metadata.py

@@ -39,6 +39,7 @@ class TestMetadata:
 
     def test_file_metadata(self, file_metadata):
         data = FileSystemFileMetadata(file_metadata, '/')
+        assert data.id == '/code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9'
         assert data.path == '/code/website/osfstoragecache/77094244-aa24-48da-9437-d8ce6f7a94e9'
         assert data.provider == 'filesystem'
         assert data.modified == 'Wed, 20 Sep 2017 15:16:02 +0000'

tests/providers/github/test_metadata.py

@@ -20,6 +20,7 @@ def test_build_file_metadata_from_tree(self, metadata_fixtures):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == '/README.md'
         assert metadata.name == 'README.md'
         assert metadata.path == '/README.md'
         assert metadata.modified is None
@@ -45,6 +46,7 @@ def test_build_file_metadata_from_contents(self, metadata_fixtures):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == '/epsilon'
         assert metadata.name == 'epsilon'
         assert metadata.path == '/epsilon'
         assert metadata.modified is None
@@ -70,6 +72,7 @@ def test_build_folder_metadata_from_tree(self, metadata_fixtures):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == '/foldera/folderb/lorch/'
         assert metadata.name == 'lorch'
         assert metadata.path == '/foldera/folderb/lorch/'
         assert metadata.extra == {}
@@ -84,6 +87,7 @@ def test_build_folder_metadata_from_content(self, metadata_fixtures):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == '/manyfiles/'
         assert metadata.name == 'manyfiles'
         assert metadata.path == '/manyfiles/'
         assert metadata.extra == {}
@@ -98,6 +102,7 @@ def test_file_metadata_with_ref(self, metadata_fixtures):
         except Exception as exc:
             pytest.fail(str(exc))
 
+        assert metadata.id == '/README.md'
         assert metadata.name == 'README.md'
         assert metadata.path == '/README.md'
         assert metadata.modified is None

tests/providers/onedrive/test_metadata.py

@@ -23,6 +23,7 @@ def test_build_file_metadata(self, root_provider_fixtures):
             'etag': 'aRjRENTBFNDAwREZFN0Q0RSEyOTEuMg',
             'webView': 'https://1drv.ms/t/s!AE59_g1ADtX0giM',
         }
+        assert metadata.id == 'F4D50E400DFE7D4E!291'
         assert metadata.name == 'toes.txt'
         assert metadata.path == '/{}'.format(root_provider_fixtures['file_id'])
         assert metadata.size == 11
@@ -45,6 +46,7 @@ def test_build_folder_metadata(self, root_provider_fixtures):
 
         metadata = OneDriveFolderMetadata(root_provider_fixtures['folder_metadata'], od_path)
         assert metadata.provider == 'onedrive'
+        assert metadata.id == 'F4D50E400DFE7D4E!290'
         assert metadata.name == 'teeth'
         assert metadata.path == '/F4D50E400DFE7D4E!290/'
         assert metadata.etag == 'aRjRENTBFNDAwREZFN0Q0RSEyOTAuMA'

tests/providers/osfstorage/test_metadata.py

@@ -13,6 +13,7 @@ class TestFileMetadata:
     def test_file_metadata(self, file_metadata, file_metadata_object):
 
         assert file_metadata_object.provider == 'osfstorage'
+        assert file_metadata_object.id == '59a9b628b7d1c903ab5a8f52'
         assert file_metadata_object.name == 'doc.rst'
         assert file_metadata_object.path == '/59a9b628b7d1c903ab5a8f52'
         assert str(file_metadata_object.materialized_path) == '/doc.rst'
@@ -69,6 +70,7 @@ class TestFolderMetadata:
     def test_folder_metadata(self, folder_metadata_object):
 
         assert folder_metadata_object.provider == 'osfstorage'
+        assert folder_metadata_object.id == '59c0054cb7d1c90114c456af'
         assert folder_metadata_object.name == 'New Folder'
         assert folder_metadata_object.path == '/59c0054cb7d1c90114c456af/'
         assert str(folder_metadata_object.materialized_path) == '/New Folder/'

tests/providers/owncloud/test_metadata.py

@@ -16,6 +16,7 @@ class TestFileMetadata:
 
     def test_file_metadata(self, file_metadata_object):
         assert file_metadata_object.provider == 'owncloud'
+        assert file_metadata_object.id == '/Documents/dissertation.aux'
         assert file_metadata_object.name == 'dissertation.aux'
         assert file_metadata_object.path == '/Documents/dissertation.aux'
         assert file_metadata_object.materialized_path == '/Documents/dissertation.aux'
@@ -42,6 +43,7 @@ def test_file_metadata(self, file_metadata_object):
 
     def test_file_metadata_less_info(self, file_metadata_object_less_info):
         assert file_metadata_object_less_info.provider == 'owncloud'
+        assert file_metadata_object_less_info.id == '/Documents/dissertation.aux'
         assert file_metadata_object_less_info.name == 'dissertation.aux'
         assert file_metadata_object_less_info.path == '/Documents/dissertation.aux'
         assert file_metadata_object_less_info.materialized_path == '/Documents/dissertation.aux'
@@ -70,6 +72,7 @@ class TestFolderMetadata:
 
     def test_folder_metadata(self, folder_metadata_object):
         assert folder_metadata_object.provider == 'owncloud'
+        assert folder_metadata_object.id == '/'
         assert folder_metadata_object.name == 'Documents'
         assert folder_metadata_object.path == '/'
         assert folder_metadata_object.materialized_path == '/'
@@ -92,6 +95,7 @@ def test_folder_metadata(self, folder_metadata_object):
     def test_folder_metadata_less_info(self, folder_metadata_object_less_info):
 
         assert folder_metadata_object_less_info.provider == 'owncloud'
+        assert folder_metadata_object_less_info.id == '/'
         assert folder_metadata_object_less_info.name == 'Documents'
         assert folder_metadata_object_less_info.path == '/'
         assert folder_metadata_object_less_info.materialized_path == '/'

tests/providers/s3/test_metadata.py

@@ -15,6 +15,7 @@ class TestFileMetadataHeaders:
     def test_file_metadata_headers(self, file_metadata_headers_object, file_header_metadata):
         assert not file_metadata_headers_object.is_folder
         assert file_metadata_headers_object.is_file
+        assert file_metadata_headers_object.id == '/test-path'
         assert file_metadata_headers_object.name == 'test-path'
         assert file_metadata_headers_object.path == '/test-path'
         assert file_metadata_headers_object.materialized_path == '/test-path'
@@ -46,6 +47,7 @@ def test_file_metadata(self, file_metadata_object):
         assert file_metadata_object.is_file
         assert file_metadata_object.kind == 'file'
 
+        assert file_metadata_object.id == '/my-image.jpg'
         assert file_metadata_object.name == 'my-image.jpg'
         assert file_metadata_object.path == '/my-image.jpg'
         assert file_metadata_object.materialized_path == '/my-image.jpg'
@@ -81,6 +83,7 @@ def test_folder_metadata(self, folder_metadata_object):
         assert not folder_metadata_object.is_file
         assert folder_metadata_object.kind == 'folder'
 
+        assert folder_metadata_object.id == '/photos/'
         assert folder_metadata_object.name == 'photos'
         assert folder_metadata_object.path == '/photos/'
         assert folder_metadata_object.materialized_path == '/photos/'
@@ -103,6 +106,7 @@ def test_folder_key_metadata(self, folder_key_metadata_object):
         assert not folder_key_metadata_object.is_file
         assert folder_key_metadata_object.kind == 'folder'
 
+        assert folder_key_metadata_object.id == '/naptime/'
         assert folder_key_metadata_object.name == 'naptime'
         assert folder_key_metadata_object.path == '/naptime/'
 

tests/utils.py

@@ -26,6 +26,7 @@ async def __call__(self, *args, **kwargs):
 
 
 class MockFileMetadata(metadata.BaseFileMetadata):
+    id = '/Foo.name'
     provider = 'MockProvider'
     name = 'Foo.name'
     size = 1337
@@ -41,6 +42,7 @@ def __init__(self):
 
 
 class MockFolderMetadata(metadata.BaseFolderMetadata):
+    id = '/Foo.name'
     provider = 'MockProvider'
     name = 'Bar'
     size = 1337

waterbutler/core/metadata.py

@@ -133,6 +133,16 @@ def kind(self) -> str:
         """ `file` or `folder` """
         raise NotImplementedError
 
+    @property
+    @abc.abstractmethod
+    def id(self) -> str:
+        """ This property consolidates path-based and id-based file/folder/object identity under a
+        single one: ``id``, which makes inter provider move/copy actions easier.  For path-based
+        providers, the value is identical to ``path`` property.  For id-based providers, it is the
+        raw identity by definition (``self.raw['id']`` for most providers).
+        """
+        raise NotImplementedError
+
     @property
     @abc.abstractmethod
     def name(self) -> str: