Merge pull request natcap#1428 from emlys/bugfix/1308

Prompt user to extract datastack archive to permanent location

HISTORY.rst

@@ -42,6 +42,8 @@ Unreleased Changes
       have been replaced with ``numpy.prod``.
       https://github.com/natcap/invest/issues/1410
     * Add support for python 3.11 (`#1103 <https://github.com/natcap/invest/issues/1103>`_)
+    * Datastack archives will now be correctly extracted
+      (`#1308 <https://github.com/natcap/invest/issues/1308>`_)
 * NDR
     * Fixing an issue where minor geometric issues in the watersheds input
       (such as a ring self-intersection) would raise an error in the model.

src/natcap/invest/datastack.py

@@ -170,12 +170,15 @@ def format_args_dict(args_dict, model_name):
     return args_string
 
 
-def get_datastack_info(filepath):
+def get_datastack_info(filepath, extract_path=None):
     """Get information about a datastack.
 
     Args:
         filepath (string): The path to a file on disk that can be extracted as
             a datastack, parameter set, or logfile.
+        extract_path (str): Path to a directory to extract the datastack, if
+            provided as an archive. Will be overwritten if it already exists,
+            or created if it does not already exist.
 
     Returns:
         A 2-tuple.  The first item of the tuple is one of:
@@ -188,23 +191,18 @@ def get_datastack_info(filepath):
         parsed args, modelname and invest version that the file was built with.
     """
     if tarfile.is_tarfile(filepath):
+        if not extract_path:
+            raise ValueError('extract_path must be provided if using archive')
+        if os.path.isfile(extract_path):
+            os.remove(extract_path)
+        elif os.path.isdir(extract_path):
+            shutil.rmtree(extract_path)
+        os.mkdir(extract_path)
         # If it's a tarfile, we need to extract the parameters file to be able
         # to inspect the parameters and model details.
-        with tarfile.open(filepath) as archive:
-            try:
-                temp_directory = tempfile.mkdtemp()
-                archive.extract('./' + DATASTACK_PARAMETER_FILENAME,
-                                temp_directory)
-                return 'archive', extract_parameter_set(
-                    os.path.join(temp_directory, DATASTACK_PARAMETER_FILENAME))
-            finally:
-                try:
-                    shutil.rmtree(temp_directory)
-                except OSError:
-                    # If something happens and we can't remove temp_directory,
-                    # just log the exception and continue with program
-                    # execution.
-                    LOGGER.exception('Could not remove %s', temp_directory)
+        extract_datastack_archive(filepath, extract_path)
+        return 'archive', extract_parameter_set(
+            os.path.join(extract_path, DATASTACK_PARAMETER_FILENAME))
 
     try:
         return 'json', extract_parameter_set(filepath)

src/natcap/invest/ui_server.py

@@ -149,9 +149,9 @@ def post_datastack_file():
     Returns:
         A JSON string.
     """
-    filepath = request.get_json()
+    payload = request.get_json()
     stack_type, stack_info = datastack.get_datastack_info(
-        filepath)
+        payload['filepath'], payload.get('extractPath', None))
     model_name = PYNAME_TO_MODEL_NAME_MAP[stack_info.model_name]
     result_dict = {
         'type': stack_type,

tests/test_datastack.py

@@ -610,7 +610,8 @@ def test_get_datastack_info_archive(self):
         datastack.build_datastack_archive(
             params, 'test_datastack_modules.simple_parameters', archive_path)
 
-        stack_type, stack_info = datastack.get_datastack_info(archive_path)
+        stack_type, stack_info = datastack.get_datastack_info(
+            archive_path, extract_path=os.path.join(self.workspace, 'archive'))
 
         self.assertEqual(stack_type, 'archive')
         self.assertEqual(stack_info, datastack.ParameterSet(

tests/test_ui_server.py

@@ -102,7 +102,7 @@ def test_post_datastack_file(self):
         with open(filepath, 'w') as file:
             file.write(json.dumps(expected_datastack))
         response = test_client.post(
-            f'{ROUTE_PREFIX}/post_datastack_file', json=filepath)
+            f'{ROUTE_PREFIX}/post_datastack_file', json={'filepath': filepath})
         response_data = json.loads(response.get_data(as_text=True))
         self.assertEqual(
             set(response_data),

workbench/src/renderer/components/SetupTab/index.jsx

@@ -278,7 +278,21 @@ class SetupTab extends React.Component {
     const { pyModuleName, switchTabs, t } = this.props;
     let datastack;
     try {
-      datastack = await fetchDatastackFromFile(filepath);
+      if (filepath.endsWith('gz')) {  // .tar.gz, .tgz
+        const extractLocation = await ipcRenderer.invoke(
+          ipcMainChannels.SHOW_SAVE_DIALOG,
+          { title: t('Choose location to extract archive') }
+        );
+        if (extractLocation.filePath) {
+          datastack = await fetchDatastackFromFile({
+            filepath: filepath,
+            extractPath: extractLocation.filePath});
+        } else {
+          return;
+        }
+      } else {
+          datastack = await fetchDatastackFromFile({ filepath: filepath });
+      }
     } catch (error) {
       logger.error(error);
       alert( // eslint-disable-line no-alert

workbench/tests/invest/flaskapp.test.js

@@ -101,7 +101,8 @@ describe('requests to flask endpoints', () => {
     });
 
     // Second test the datastack is read and parsed
-    const data2 = await server_requests.fetchDatastackFromFile(filepath);
+    const data2 = await server_requests.fetchDatastackFromFile(
+      { filepath: filepath });
     const expectedKeys2 = [
       'type',
       'args',