diff --git a/girder_worker_utils/tests/contrib/__init__.py b/girder_worker_utils/tests/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/girder_worker_utils/tests/contrib/girder_io_test.py b/girder_worker_utils/tests/contrib/girder_io_test.py new file mode 100644 index 0000000..7d2e899 --- /dev/null +++ b/girder_worker_utils/tests/contrib/girder_io_test.py @@ -0,0 +1,49 @@ +import girder_client +import mock +import os +import pytest + +from girder_worker_utils.transforms.contrib import girder_io + + +@pytest.fixture +def mock_gc(): + return mock.MagicMock(spec=girder_client.GirderClient) + + +@pytest.fixture +def mock_rmtree(): + with mock.patch('shutil.rmtree') as rmtree: + yield rmtree + + +def test_GirderFileIdAllowDirect_without_env(mock_gc, mock_rmtree): + local_path = os.path.abspath(__file__) + t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', local_path, gc=mock_gc) + t.transform() + mock_gc.downloadFile.assert_called_once() + assert 'the_id' in mock_gc.downloadFile.call_args[0] + mock_rmtree.assert_not_called() + t.cleanup() + mock_rmtree.assert_called_once() + + +@mock.patch.dict(os.environ, {'GW_DIRECT_PATHS': 'true'}) +def test_GirderFileIdAllowDirect_with_env(mock_gc, mock_rmtree): + local_path = os.path.abspath(__file__) + t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', local_path, gc=mock_gc) + t.transform() + mock_gc.downloadFile.assert_not_called() + t.cleanup() + mock_rmtree.assert_not_called() + + +@mock.patch.dict(os.environ, {'GW_DIRECT_PATHS': 'true'}) +def test_GirderFileIdAllowDirect_with_env_and_unreachable_file(mock_gc, mock_rmtree): + t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', 'the_path', gc=mock_gc) + t.transform() + mock_gc.downloadFile.assert_called_once() + assert 'the_id' in mock_gc.downloadFile.call_args[0] + mock_rmtree.assert_not_called() + t.cleanup() + mock_rmtree.assert_called_once() diff --git a/girder_worker_utils/tests/girder_io_test.py b/girder_worker_utils/tests/girder_io_test.py index 8e7abe0..3bfa786 100644 --- a/girder_worker_utils/tests/girder_io_test.py +++ b/girder_worker_utils/tests/girder_io_test.py @@ -99,3 +99,13 @@ def test_GirderUploadJobArtifact(mock_gc): urls = sorted(args[0][0] for args in mock_gc.post.call_args_list) assert 'name=file1.txt' in urls[0] assert 'name=file2.txt' in urls[1] + + +def test_GirderFileId(mock_gc, mock_rmtree): + t = girder_io.GirderFileId(_id='the_id', gc=mock_gc) + t.transform() + mock_gc.downloadFile.assert_called_once() + assert 'the_id' in mock_gc.downloadFile.call_args[0] + mock_rmtree.assert_not_called() + t.cleanup() + mock_rmtree.assert_called_once() diff --git a/girder_worker_utils/transforms/contrib/__init__.py b/girder_worker_utils/transforms/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/girder_worker_utils/transforms/contrib/girder_io.py b/girder_worker_utils/transforms/contrib/girder_io.py new file mode 100644 index 0000000..91c7b3e --- /dev/null +++ b/girder_worker_utils/transforms/contrib/girder_io.py @@ -0,0 +1,76 @@ +import os +import shutil +import tempfile + +from ..girder_io import GirderClientTransform + + +class GirderFileIdAllowDirect(GirderClientTransform): + """ + This transform either uses direct path to access a file, if possible and + allowed, or downloads a Girder File to the local machine and passes its + local path into the function. The direct path is only used if the + GW_DIRECT_PATHS environment variable is set. + + WARNING: if a direct path is used, the task MUST NOT modify the file. It + is the resposibility of the user of this transform to ensure tasks treat + files as read-only. + + To use this transform from Girder, it should be called via something like: + ``` + try: + local_path = File().getLocalFilePath(file) + except FilePathException: + local_path = None + input_path = GirderFileIdAllowDirect( + str(file['_id']), file['name'], local_path) + ``` + + :param _id: The ID of the file to download. + :type _id: str + :param name: The name of the file. If the file must be downloaded, the + extension is preserved. + :type name: str + :param local_path: If specified and the path exists and is reachable by + after the transform, the file is accessed directly. + :type local_path: str + """ + def __init__(self, _id, name='', local_path=None, **kwargs): + super(GirderFileIdAllowDirect, self).__init__(**kwargs) + self.file_id = _id + self.file_name = name + self.local_file_path = local_path + + def _repr_model_(self): + if self.local_file_path: + return '{}({!r}, {!r}, {!r})'.format( + self.__class__.__name__, self.file_id, self.file_name, self.local_file_path) + return '{}({!r}, {!r})'.format(self.__class__.__name__, self.file_id, self.file_name) + + def _allowDirectPath(self): + """ + Check if the worker environment permits direct paths. This just checks + if the environment variable GW_DIRECT_PATHS is set to a non-empty + value. + + :returns: True if direct paths are allowed. + """ + return bool(os.environ.get('GW_DIRECT_PATHS')) + + def transform(self): + # Don't download if self.local_file_path is set and direct paths are + # allowed. + if (self.local_file_path and self._allowDirectPath() and + os.path.isfile(self.local_file_path)): + self.temp_dir_path = None + self.file_path = self.local_file_path + else: + self.temp_dir_path = tempfile.mkdtemp() + self.file_path = os.path.join(self.temp_dir_path, '{}{}'.format( + self.file_id, os.path.splitext(self.file_name)[1])) + self.gc.downloadFile(self.file_id, self.file_path) + return self.file_path + + def cleanup(self): + if self.temp_dir_path: + shutil.rmtree(self.temp_dir_path, ignore_errors=True) diff --git a/tox.ini b/tox.ini index c9bf658..cacc213 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = pytest --cov=girder_worker_utils --cov-report html --cov-report term enable-extensions = C, D, E, F, I, N, W max-line-length = 100 max-complexity = 10 -ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400 +ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400,W504 import-order-style = google application-import-names = girder_worker_utils,gw_utils_demo_app