Move the direct path transform to a contrib package.

Add tests for the GirderFileId transform and for the new
contrib.GirderFileIdAllowDirect transform.

girder_worker_utils/tests/contrib/girder_io_test.py

@@ -0,0 +1,55 @@
+import os
+
+import girder_client
+import mock
+import pytest
+
+from girder_worker_utils.transforms.contrib import girder_io
+
+
+@pytest.fixture
+def mock_file_load():
+    from girder.models.file import File
+    with mock.patch.object(File(), 'load') as load:
+        load.return_value = {'name': 'the_name'}
+        yield load
+
+
+@pytest.fixture
+def mock_file_getLocalFilePath():
+    from girder.models.file import File
+    with mock.patch.object(File(), 'getLocalFilePath') as getLocalFilePath:
+        getLocalFilePath.return_value = os.path.realpath(__file__)
+        yield getLocalFilePath
+
+
+@pytest.fixture
+def mock_gc():
+    return mock.MagicMock(spec=girder_client.GirderClient)
+
+
+@pytest.fixture
+def mock_rmtree():
+    with mock.patch('shutil.rmtree') as rmtree:
+        yield rmtree
+
+
+def test_GirderFileIdAllowDirect_without_env(
+        mock_gc, mock_rmtree, mock_file_load, mock_file_getLocalFilePath):
+    t = girder_io.GirderFileIdAllowDirect(_id='the_id', gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_called_once()
+    assert 'the_id' in mock_gc.downloadFile.call_args[0]
+    mock_rmtree.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_called_once()
+
+
+@mock.patch.dict(os.environ, {'GW_DIRECT_PATHS': 'true'})
+def test_GirderFileIdAllowDirect_with_env(
+        mock_gc, mock_rmtree, mock_file_load, mock_file_getLocalFilePath):
+    t = girder_io.GirderFileIdAllowDirect(_id='the_id', gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_not_called()

girder_worker_utils/tests/girder_io_test.py

@@ -99,3 +99,13 @@ def test_GirderUploadJobArtifact(mock_gc):
     urls = sorted(args[0][0] for args in mock_gc.post.call_args_list)
     assert 'name=file1.txt' in urls[0]
     assert 'name=file2.txt' in urls[1]
+
+
+def test_GirderFileId(mock_gc, mock_rmtree):
+    t = girder_io.GirderFileId(_id='the_id', gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_called_once()
+    assert 'the_id' in mock_gc.downloadFile.call_args[0]
+    mock_rmtree.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_called_once()

girder_worker_utils/transforms/contrib/girder_io.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+import tempfile
+
+from ..girder_io import GirderClientTransform
+
+
+class GirderFileIdAllowDirect(GirderClientTransform):
+    """
+    This transform either uses direct path to access a file, if possible and
+    allowed, or downloads a Girder File to the local machine and passes its
+    local path into the function.
+
+    WARNING: if a direct path is used, the task MUST NOT modify the file.  It
+    is the resposibility of the user of this transform to ensure tasks treat
+    files as read-only.
+
+    :param _id: The ID of the file to download.
+    :type _id: str
+    """
+    def __init__(self, _id, **kwargs):
+        super(GirderFileIdAllowDirect, self).__init__(**kwargs)
+        from girder.models.file import File
+        from girder.exceptions import FilePathException
+        self.file_id = _id
+        file = File().load(self.file_id, force=True)
+        # Store the original file name so that, if downloading the file, the
+        # extension can be preserved.
+        self.file_name = file['name']
+        try:
+            # Add a local file path if direct paths are allowed
+            self.local_file_path = File().getLocalFilePath(file)
+        except FilePathException:
+            self.local_file_path = None
+
+    def _repr_model_(self):
+        if self.local_file_path:
+            return "{}({!r}) - {!r} - {!r}".format(
+                self.__class__.__name__, self.file_id, self.file_name, self.local_file_path)
+        return "{}({!r}) - {!r}".format(self.__class__.__name__, self.file_id, self.file_name)
+
+    def _allowDirectPath(self):
+        """
+        Check if the worker environment permits direct paths.  This just checks
+        if the environment variable GW_DIRECT_PATHS is set to a non-empry
+        value.
+        """
+        if os.environ.get('GW_DIRECT_PATHS'):
+            return True
+        return False
+
+    def transform(self):
+        # Don't download if self.local_file_path is set and direct paths are
+        # allowed.
+        if (self.local_file_path and self._allowDirectPath() and
+                os.path.isfile(self.local_file_path)):
+            self.temp_dir_path = None
+            self.file_path = self.local_file_path
+        else:
+            self.temp_dir_path = tempfile.mkdtemp()
+            self.file_path = os.path.join(self.temp_dir_path, '{}{}'.format(
+                self.file_id, os.path.splitext(self.file_name)[1]))
+            self.gc.downloadFile(self.file_id, self.file_path)
+        return self.file_path
+
+    def cleanup(self):
+        if self.temp_dir_path:
+            shutil.rmtree(self.temp_dir_path, ignore_errors=True)

girder_worker_utils/transforms/girder_io.py

@@ -52,46 +52,21 @@ class GirderFileId(GirderClientTransform):
     def __init__(self, _id, **kwargs):
         super(GirderFileId, self).__init__(**kwargs)
         self.file_id = _id
-        # Add a local file path if direct paths are allowed
-        try:
-            from girder.models.file import File
-            from girder.models.setting import Setting
-            from girder.exceptions import FilePathException
-            try:
-                from girder_worker.girder_plugin.constants import PluginSettings
-            except ImportError:
-                from girder.plugins.worker.constants import PluginSettings
-            if Setting().get(PluginSettings.DIRECT_PATH):
-                try:
-                    self.local_file_path = File().getLocalFilePath(
-                        File().load(self.file_id, force=True))
-                except FilePathException:
-                    pass
-        except ImportError:
-            pass
 
     def _repr_model_(self):
         return "{}('{}')".format(self.__class__.__name__, self.file_id)
 
     def transform(self):
-        import girder_worker.utils
-
-        # Don't download if self.local_file_path is set and direct paths are
-        # allowed.
-        if (getattr(self, 'local_file_path', None) and
-                girder_worker.utils.allow_direct_path() and
-                os.path.isfile(self.local_file_path)):
-            self.file_path = self.local_file_path
-            self.temp_dir_path = None
-        else:
-            self.temp_dir_path = tempfile.mkdtemp(dir=girder_worker.utils.get_tmp_root())
-            self.file_path = os.path.join(self.temp_dir_path, '{}'.format(self.file_id))
-            self.gc.downloadFile(self.file_id, self.file_path)
+        self.file_path = os.path.join(
+            tempfile.mkdtemp(), '{}'.format(self.file_id))
+
+        self.gc.downloadFile(self.file_id, self.file_path)
+
         return self.file_path
 
     def cleanup(self):
-        if self.temp_dir_path:
-            shutil.rmtree(self.temp_dir_path, ignore_errors=True)
+        shutil.rmtree(os.path.dirname(self.file_path),
+                      ignore_errors=True)
 
 
 class GirderItemMetadata(GirderClientTransform):

requirements-dev.in

@@ -2,6 +2,7 @@ flake8
 flake8-blind-except
 flake8-docstrings
 flake8-import-order
+girder
 mock
 pytest
 pytest-cov

requirements.txt

@@ -4,30 +4,64 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in requirements-dev.in
 #
+backports.functools-lru-cache==1.5  # via cheroot, jaraco.functools
+bcrypt==3.1.4             # via girder
+boto3==1.9.38             # via girder
+botocore==1.12.38         # via boto3, s3transfer
 certifi==2017.7.27.1      # via requests
+cffi==1.11.5              # via bcrypt
 chardet==3.0.4            # via requests
-click==6.7                # via girder-client
+cheroot==6.5.2            # via cherrypy
+cherrypy==11.0.0          # via girder
+click-plugins==1.0.4      # via girder
+click==6.7                # via click-plugins, girder, girder-client
+configparser==3.5.0       # via flake8
 coverage==4.4.1           # via pytest-cov
 diskcache==2.9.0          # via girder-client
+docutils==0.14            # via botocore
+dogpile.cache==0.6.7      # via girder
+enum34==1.1.6             # via flake8
+filelock==3.0.10          # via girder
 flake8-blind-except==0.1.1
 flake8-docstrings==1.1.0
 flake8-import-order==0.13
 flake8-polyfill==1.0.1    # via flake8-docstrings
 flake8==3.4.1
 funcsigs==1.0.2 ; python_version < "3.5"
+functools32==3.2.3.post2  # via jsonschema
+futures==3.2.0            # via s3transfer
 girder-client==2.3.0
+girder==2.5.0
 idna==2.6                 # via requests
+jaraco.functools==1.20    # via tempora
+jmespath==0.9.3           # via boto3, botocore
+jsonpickle==1.0
+jsonschema==2.6.0         # via girder
+mako==1.0.7               # via girder
+markupsafe==1.1.0         # via mako
 mccabe==0.6.1             # via flake8
 mock==2.0.0
+more-itertools==4.3.0     # via cheroot, jaraco.functools
 pbr==3.1.1                # via mock
+portend==2.3              # via cherrypy
+psutil==5.4.8             # via girder
 py==1.4.34                # via pytest
 pycodestyle==2.3.1        # via flake8, flake8-import-order
+pycparser==2.19           # via cffi
 pydocstyle==2.0.0         # via flake8-docstrings
 pyflakes==1.5.0           # via flake8
+pymongo==3.7.2            # via girder
 pytest-cov==2.5.1
 pytest==3.2.3
+python-dateutil==2.6.1    # via botocore, girder
+pytz==2018.7              # via girder, tempora
+pyyaml==3.13              # via girder
 requests-toolbelt==0.8.0  # via girder-client
-requests==2.18.4          # via girder-client, requests-toolbelt
+requests==2.18.4          # via girder, girder-client, requests-toolbelt
+s3transfer==0.1.13        # via boto3
+setuptools-scm==3.1.0
+shutilwhich==1.1.0        # via girder
 six==1.11.0
 snowballstemmer==1.2.1    # via pydocstyle
-urllib3==1.22             # via requests
+tempora==1.14             # via portend
+urllib3==1.22             # via botocore, requests

tox.ini

@@ -26,7 +26,7 @@ commands = pytest --cov=girder_worker_utils --cov-report html --cov-report term
 enable-extensions = C, D, E, F, I, N, W
 max-line-length = 100
 max-complexity = 10
-ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400
+ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400,W504
 import-order-style = google
 application-import-names = girder_worker_utils,gw_utils_demo_app