diff --git a/HISTORY.rst b/HISTORY.rst index 59523f494b..8eeea886da 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -42,6 +42,8 @@ Unreleased Changes have been replaced with ``numpy.prod``. https://github.com/natcap/invest/issues/1410 * Add support for python 3.11 (`#1103 `_) + * Datastack archives will now be correctly extracted + (`#1308 `_) * NDR * Fixing an issue where minor geometric issues in the watersheds input (such as a ring self-intersection) would raise an error in the model. diff --git a/src/natcap/invest/datastack.py b/src/natcap/invest/datastack.py index b3dac244e6..4a9cd52d4a 100644 --- a/src/natcap/invest/datastack.py +++ b/src/natcap/invest/datastack.py @@ -170,12 +170,15 @@ def format_args_dict(args_dict, model_name): return args_string -def get_datastack_info(filepath): +def get_datastack_info(filepath, extract_path=None): """Get information about a datastack. Args: filepath (string): The path to a file on disk that can be extracted as a datastack, parameter set, or logfile. + extract_path (str): Path to a directory to extract the datastack, if + provided as an archive. Will be overwritten if it already exists, + or created if it does not already exist. Returns: A 2-tuple. The first item of the tuple is one of: @@ -188,23 +191,18 @@ def get_datastack_info(filepath): parsed args, modelname and invest version that the file was built with. """ if tarfile.is_tarfile(filepath): + if not extract_path: + raise ValueError('extract_path must be provided if using archive') + if os.path.isfile(extract_path): + os.remove(extract_path) + elif os.path.isdir(extract_path): + shutil.rmtree(extract_path) + os.mkdir(extract_path) # If it's a tarfile, we need to extract the parameters file to be able # to inspect the parameters and model details. - with tarfile.open(filepath) as archive: - try: - temp_directory = tempfile.mkdtemp() - archive.extract('./' + DATASTACK_PARAMETER_FILENAME, - temp_directory) - return 'archive', extract_parameter_set( - os.path.join(temp_directory, DATASTACK_PARAMETER_FILENAME)) - finally: - try: - shutil.rmtree(temp_directory) - except OSError: - # If something happens and we can't remove temp_directory, - # just log the exception and continue with program - # execution. - LOGGER.exception('Could not remove %s', temp_directory) + extract_datastack_archive(filepath, extract_path) + return 'archive', extract_parameter_set( + os.path.join(extract_path, DATASTACK_PARAMETER_FILENAME)) try: return 'json', extract_parameter_set(filepath) diff --git a/src/natcap/invest/ui_server.py b/src/natcap/invest/ui_server.py index 60955d47d2..bfbe1874ec 100644 --- a/src/natcap/invest/ui_server.py +++ b/src/natcap/invest/ui_server.py @@ -149,9 +149,9 @@ def post_datastack_file(): Returns: A JSON string. """ - filepath = request.get_json() + payload = request.get_json() stack_type, stack_info = datastack.get_datastack_info( - filepath) + payload['filepath'], payload.get('extractPath', None)) model_name = PYNAME_TO_MODEL_NAME_MAP[stack_info.model_name] result_dict = { 'type': stack_type, diff --git a/tests/test_datastack.py b/tests/test_datastack.py index d162c61f04..33a01de2f4 100644 --- a/tests/test_datastack.py +++ b/tests/test_datastack.py @@ -610,7 +610,8 @@ def test_get_datastack_info_archive(self): datastack.build_datastack_archive( params, 'test_datastack_modules.simple_parameters', archive_path) - stack_type, stack_info = datastack.get_datastack_info(archive_path) + stack_type, stack_info = datastack.get_datastack_info( + archive_path, extract_path=os.path.join(self.workspace, 'archive')) self.assertEqual(stack_type, 'archive') self.assertEqual(stack_info, datastack.ParameterSet( diff --git a/tests/test_ui_server.py b/tests/test_ui_server.py index 32c907260a..426dc57005 100644 --- a/tests/test_ui_server.py +++ b/tests/test_ui_server.py @@ -102,7 +102,7 @@ def test_post_datastack_file(self): with open(filepath, 'w') as file: file.write(json.dumps(expected_datastack)) response = test_client.post( - f'{ROUTE_PREFIX}/post_datastack_file', json=filepath) + f'{ROUTE_PREFIX}/post_datastack_file', json={'filepath': filepath}) response_data = json.loads(response.get_data(as_text=True)) self.assertEqual( set(response_data), diff --git a/workbench/src/renderer/components/SetupTab/index.jsx b/workbench/src/renderer/components/SetupTab/index.jsx index 0485d0ed71..c241dd5706 100644 --- a/workbench/src/renderer/components/SetupTab/index.jsx +++ b/workbench/src/renderer/components/SetupTab/index.jsx @@ -278,7 +278,21 @@ class SetupTab extends React.Component { const { pyModuleName, switchTabs, t } = this.props; let datastack; try { - datastack = await fetchDatastackFromFile(filepath); + if (filepath.endsWith('gz')) { // .tar.gz, .tgz + const extractLocation = await ipcRenderer.invoke( + ipcMainChannels.SHOW_SAVE_DIALOG, + { title: t('Choose location to extract archive') } + ); + if (extractLocation.filePath) { + datastack = await fetchDatastackFromFile({ + filepath: filepath, + extractPath: extractLocation.filePath}); + } else { + return; + } + } else { + datastack = await fetchDatastackFromFile({ filepath: filepath }); + } } catch (error) { logger.error(error); alert( // eslint-disable-line no-alert diff --git a/workbench/tests/invest/flaskapp.test.js b/workbench/tests/invest/flaskapp.test.js index aacd522e9c..9a8d7309f4 100644 --- a/workbench/tests/invest/flaskapp.test.js +++ b/workbench/tests/invest/flaskapp.test.js @@ -101,7 +101,8 @@ describe('requests to flask endpoints', () => { }); // Second test the datastack is read and parsed - const data2 = await server_requests.fetchDatastackFromFile(filepath); + const data2 = await server_requests.fetchDatastackFromFile( + { filepath: filepath }); const expectedKeys2 = [ 'type', 'args',