Support using local files if appropriately configured.

girder_worker_utils/tests/contrib/girder_io_test.py

@@ -0,0 +1,49 @@
+import girder_client
+import mock
+import os
+import pytest
+
+from girder_worker_utils.transforms.contrib import girder_io
+
+
+@pytest.fixture
+def mock_gc():
+    return mock.MagicMock(spec=girder_client.GirderClient)
+
+
+@pytest.fixture
+def mock_rmtree():
+    with mock.patch('shutil.rmtree') as rmtree:
+        yield rmtree
+
+
+def test_GirderFileIdAllowDirect_without_env(mock_gc, mock_rmtree):
+    local_path = os.path.abspath(__file__)
+    t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', local_path, gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_called_once()
+    assert 'the_id' in mock_gc.downloadFile.call_args[0]
+    mock_rmtree.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_called_once()
+
+
+@mock.patch.dict(os.environ, {'GW_DIRECT_PATHS': 'true'})
+def test_GirderFileIdAllowDirect_with_env(mock_gc, mock_rmtree):
+    local_path = os.path.abspath(__file__)
+    t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', local_path, gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_not_called()
+
+
+@mock.patch.dict(os.environ, {'GW_DIRECT_PATHS': 'true'})
+def test_GirderFileIdAllowDirect_with_env_and_unreachable_file(mock_gc, mock_rmtree):
+    t = girder_io.GirderFileIdAllowDirect('the_id', 'the_name', 'the_path', gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_called_once()
+    assert 'the_id' in mock_gc.downloadFile.call_args[0]
+    mock_rmtree.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_called_once()

girder_worker_utils/tests/girder_io_test.py

@@ -99,3 +99,13 @@ def test_GirderUploadJobArtifact(mock_gc):
     urls = sorted(args[0][0] for args in mock_gc.post.call_args_list)
     assert 'name=file1.txt' in urls[0]
     assert 'name=file2.txt' in urls[1]
+
+
+def test_GirderFileId(mock_gc, mock_rmtree):
+    t = girder_io.GirderFileId(_id='the_id', gc=mock_gc)
+    t.transform()
+    mock_gc.downloadFile.assert_called_once()
+    assert 'the_id' in mock_gc.downloadFile.call_args[0]
+    mock_rmtree.assert_not_called()
+    t.cleanup()
+    mock_rmtree.assert_called_once()

girder_worker_utils/transforms/contrib/girder_io.py

@@ -0,0 +1,76 @@
+import os
+import shutil
+import tempfile
+
+from ..girder_io import GirderClientTransform
+
+
+class GirderFileIdAllowDirect(GirderClientTransform):
+    """
+    This transform either uses direct path to access a file, if possible and
+    allowed, or downloads a Girder File to the local machine and passes its
+    local path into the function.  The direct path is only used if the
+    GW_DIRECT_PATHS environment variable is set.
+
+    WARNING: if a direct path is used, the task MUST NOT modify the file.  It
+    is the resposibility of the user of this transform to ensure tasks treat
+    files as read-only.
+
+    To use this transform from Girder, it should be called via something like:
+    ```
+    try:
+        local_path = File().getLocalFilePath(file)
+    except FilePathException:
+        local_path = None
+    input_path = GirderFileIdAllowDirect(
+        str(file['_id']), file['name'], local_path)
+    ```
+
+    :param _id: The ID of the file to download.
+    :type _id: str
+    :param name: The name of the file.  If the file must be downloaded, the
+        extension is preserved.
+    :type name: str
+    :param local_path: If specified and the path exists and is reachable by
+        after the transform, the file is accessed directly.
+    :type local_path: str
+    """
+    def __init__(self, _id, name='', local_path=None, **kwargs):
+        super(GirderFileIdAllowDirect, self).__init__(**kwargs)
+        self.file_id = _id
+        self.file_name = name
+        self.local_file_path = local_path
+
+    def _repr_model_(self):
+        if self.local_file_path:
+            return '{}({!r}, {!r}, {!r})'.format(
+                self.__class__.__name__, self.file_id, self.file_name, self.local_file_path)
+        return '{}({!r}, {!r})'.format(self.__class__.__name__, self.file_id, self.file_name)
+
+    def _allowDirectPath(self):
+        """
+        Check if the worker environment permits direct paths.  This just checks
+        if the environment variable GW_DIRECT_PATHS is set to a non-empty
+        value.
+
+        :returns: True if direct paths are allowed.
+        """
+        return bool(os.environ.get('GW_DIRECT_PATHS'))
+
+    def transform(self):
+        # Don't download if self.local_file_path is set and direct paths are
+        # allowed.
+        if (self.local_file_path and self._allowDirectPath() and
+                os.path.isfile(self.local_file_path)):
+            self.temp_dir_path = None
+            self.file_path = self.local_file_path
+        else:
+            self.temp_dir_path = tempfile.mkdtemp()
+            self.file_path = os.path.join(self.temp_dir_path, '{}{}'.format(
+                self.file_id, os.path.splitext(self.file_name)[1]))
+            self.gc.downloadFile(self.file_id, self.file_path)
+        return self.file_path
+
+    def cleanup(self):
+        if self.temp_dir_path:
+            shutil.rmtree(self.temp_dir_path, ignore_errors=True)

tox.ini

@@ -26,7 +26,7 @@ commands = pytest --cov=girder_worker_utils --cov-report html --cov-report term
 enable-extensions = C, D, E, F, I, N, W
 max-line-length = 100
 max-complexity = 10
-ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400
+ignore = D100,D101,D102,D103,D104,D105,D107,D200,D204,D205,D400,W504
 import-order-style = google
 application-import-names = girder_worker_utils,gw_utils_demo_app