Merge pull request #812 from girder/pickle-output

Support pickle output of numpy arrays for endpoints.

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 - Add support for ellipse and circle annotations ([811](../../pull/811))
+- Support pickle output of numpy arrays for region, thumbnail, and tile_frames endpoints ([812](../../pull/812))
 
 ### Improvements
 - Improve parsing OME TIFF channel names ([806](../../pull/806))

girder/girder_large_image/rest/tiles.py

@@ -18,6 +18,7 @@
 import math
 import os
 import pathlib
+import pickle
 import re
 import urllib
 
@@ -92,6 +93,38 @@ def _handleETag(key, item, *args, **kwargs):
     setResponseHeader('Cache-control', 'max-age=3600')
 
 
+def _pickleParams(params):
+    """
+    Check if the output should be returned as pickled data and adjust the
+    parameters accordingly.
+
+    :param params: a dictionary of parameters.  If encoding starts with
+        'pickle', numpy format will be requested.
+    :return: None if the output should not be pickled.  Otherwise, the pickle
+        protocol that should be used.
+    """
+    if not str(params.get('encoding')).startswith('pickle'):
+        return None
+    params['format'] = large_image.constants.TILE_FORMAT_NUMPY
+    pickle = params['encoding'].split(':')[-1]
+    del params['encoding']
+    return int(pickle) or 4 if pickle.isdigit() else 4
+
+
+def _pickleOutput(data, protocol):
+    """
+    Pickle some data using a specific protocol and return the pickled data
+    and the recommended mime type.
+
+    :param data: the data to pickle.
+    :param protocol: the pickle protocol level.
+    :returns: the pickled data and the mime type.
+    """
+    return (
+        pickle.dumps(data, protocol=min(protocol, pickle.HIGHEST_PROTOCOL)),
+        'application/octet-stream')
+
+
 class TilesItemResource(ItemResource):
 
     def __init__(self, apiRoot):
@@ -662,8 +695,10 @@ def deleteTiles(self, item, params):
         .param('encoding', 'Output image encoding.  TILED generates a tiled '
                'tiff without the upper limit on image size the other options '
                'have.  For geospatial sources, TILED will also have '
-               'appropriate tagging.', required=False,
-               enum=['JPEG', 'PNG', 'TIFF', 'TILED'], default='JPEG')
+               'appropriate tagging.  Pickle emits python pickle data with an '
+               'optional specific protocol', required=False,
+               enum=['JPEG', 'PNG', 'TIFF', 'TILED', 'pickle', 'pickle:3',
+                     'pickle:4', 'pickle:5'], default='JPEG')
         .param('contentDisposition', 'Specify the Content-Disposition response '
                'header disposition-type value.', required=False,
                enum=['inline', 'attachment'])
@@ -691,6 +726,7 @@ def getTilesThumbnail(self, item, params):
             ('contentDispositionFileName', str)
         ])
         _handleETag('getTilesThumbnail', item, params)
+        pickle = _pickleParams(params)
         try:
             result = self.imageItemModel.getThumbnail(item, **params)
         except TileGeneralError as e:
@@ -700,6 +736,8 @@ def getTilesThumbnail(self, item, params):
         if not isinstance(result, tuple):
             return result
         thumbData, thumbMime = result
+        if pickle:
+            thumbData, thumbMime = _pickleOutput(thumbData, pickle)
         self._setContentDisposition(
             item, params.get('contentDisposition'), thumbMime, 'thumbnail',
             params.get('contentDispositionFilename'))
@@ -767,8 +805,10 @@ def getTilesThumbnail(self, item, params):
         .param('encoding', 'Output image encoding.  TILED generates a tiled '
                'tiff without the upper limit on image size the other options '
                'have.  For geospatial sources, TILED will also have '
-               'appropriate tagging.', required=False,
-               enum=['JPEG', 'PNG', 'TIFF', 'TILED'], default='JPEG')
+               'appropriate tagging.  Pickle emits python pickle data with an '
+               'optional specific protocol', required=False,
+               enum=['JPEG', 'PNG', 'TIFF', 'TILED', 'pickle', 'pickle:3',
+                     'pickle:4', 'pickle:5'], default='JPEG')
         .param('jpegQuality', 'Quality used for generating JPEG images',
                required=False, dataType='int', default=95)
         .param('jpegSubsampling', 'Chroma subsampling used for generating '
@@ -827,10 +867,13 @@ def getTilesRegion(self, item, params):
             ('contentDispositionFileName', str)
         ])
         _handleETag('getTilesRegion', item, params)
+        pickle = _pickleParams(params)
         setResponseTimeLimit(86400)
         try:
             regionData, regionMime = self.imageItemModel.getRegion(
                 item, **params)
+            if pickle:
+                regionData, regionMime = _pickleOutput(regionData, pickle)
         except TileGeneralError as e:
             raise RestException(e.args[0])
         except ValueError as e:
@@ -1175,8 +1218,10 @@ def getAssociatedImageMetadata(self, item, image, params):
         .param('encoding', 'Output image encoding.  TILED generates a tiled '
                'tiff without the upper limit on image size the other options '
                'have.  For geospatial sources, TILED will also have '
-               'appropriate tagging.', required=False,
-               enum=['JPEG', 'PNG', 'TIFF', 'TILED'], default='JPEG')
+               'appropriate tagging.  Pickle emits python pickle data with an '
+               'optional specific protocol', required=False,
+               enum=['JPEG', 'PNG', 'TIFF', 'TILED', 'pickle', 'pickle:3',
+                     'pickle:4', 'pickle:5'], default='JPEG')
         .param('jpegQuality', 'Quality used for generating JPEG images',
                required=False, dataType='int', default=95)
         .param('jpegSubsampling', 'Chroma subsampling used for generating '
@@ -1213,6 +1258,7 @@ def tileFrames(self, item, params):
 
         params = self._parseParams(params, True, self._tileFramesParams)
         _handleETag('tileFrames', item, params)
+        pickle = _pickleParams(params)
         if 'frameList' in params:
             params['frameList'] = [
                 int(f.strip()) for f in str(params['frameList']).lstrip(
@@ -1228,6 +1274,8 @@ def tileFrames(self, item, params):
         if not isinstance(result, tuple):
             return result
         regionData, regionMime = result
+        if pickle:
+            regionData, regionMime = _pickleOutput(regionData, pickle)
         self._setContentDisposition(
             item, params.get('contentDisposition'), regionMime, 'tileframes',
             params.get('contentDispositionFilename'))

girder/test_girder/test_tiles_rest.py

@@ -1,5 +1,6 @@
 import math
 import os
+import pickle
 import shutil
 import struct
 import time
@@ -821,6 +822,41 @@ def testRegions(server, admin, fsAssetstore):
     assert image[:len(utilities.BigTIFFHeader)] == utilities.BigTIFFHeader
 
 
+@pytest.mark.usefixtures('unbindLargeImage')
+@pytest.mark.plugin('large_image')
+def testRegionPickle(server, admin, fsAssetstore):
+    file = utilities.uploadExternalFile(
+        'sample_image.ptif', admin, fsAssetstore)
+    itemId = str(file['itemId'])
+
+    params = {'regionWidth': 2000, 'regionHeight': 1500,
+              'width': 500, 'height': 500,
+              'encoding': 'pickle'}
+    resp = server.request(path='/item/%s/tiles/region' % itemId,
+                          user=admin, isJson=False, params=params)
+    assert utilities.respStatus(resp) == 200
+    narray = pickle.loads(utilities.getBody(resp, text=False))
+    assert narray.shape == (375, 500, 3)
+
+    params = {'regionWidth': 2000, 'regionHeight': 1500,
+              'width': 500, 'height': 500,
+              'encoding': 'pickle:1'}
+    resp = server.request(path='/item/%s/tiles/region' % itemId,
+                          user=admin, isJson=False, params=params)
+    assert utilities.respStatus(resp) == 200
+    narray = pickle.loads(utilities.getBody(resp, text=False))
+    assert narray.shape == (375, 500, 3)
+
+    params = {'regionWidth': 2000, 'regionHeight': 1500,
+              'width': 500, 'height': 500,
+              'encoding': 'pickle:' + str(pickle.HIGHEST_PROTOCOL)}
+    resp = server.request(path='/item/%s/tiles/region' % itemId,
+                          user=admin, isJson=False, params=params)
+    assert utilities.respStatus(resp) == 200
+    narray = pickle.loads(utilities.getBody(resp, text=False))
+    assert narray.shape == (375, 500, 3)
+
+
 @pytest.mark.usefixtures('unbindLargeImage')
 @pytest.mark.plugin('large_image')
 def testPixel(server, admin, fsAssetstore):