Feat/ecoinvent interface

activity_browser/ui/wizards/db_import_wizard.py

@@ -1,16 +1,20 @@
 # -*- coding: utf-8 -*-
 import io
+import shutil
+import typing
+from functools import lru_cache
 import subprocess
 import tempfile
 import zipfile
 from pathlib import Path
 
-import eidl
+import ecoinvent_interface
 import requests
 from bw2io import BW2Package, SingleOutputEcospold2Importer
 from bw2io.extractors import Ecospold2DataExtractor
 from PySide2 import QtCore, QtWidgets
 from PySide2.QtCore import Signal, Slot
+from py7zr import py7zr
 
 from activity_browser import log
 from activity_browser.bwutils import errors
@@ -93,9 +97,14 @@ def version(self):
     def system_model(self):
         return self.ecoinvent_version_page.system_model_combobox.currentText()
 
+    @property
+    def release_type(self):
+        return self.ecoinvent_version_page.release_type_combobox.currentText()
+
     def update_downloader(self):
         self.downloader.version = self.version
         self.downloader.system_model = self.system_model
+        self.downloader.release_type = self.release_type
 
     def done(self, result: int):
         """
@@ -344,7 +353,10 @@ def __init__(self, parent=None):
         self.setLayout(layout)
 
     def initializePage(self):
-        self.stored_dbs = eidl.eidlstorage.stored_dbs
+        # TODO: get this from eco_invent
+        #   previous stored_dbs was list just listing out all the database
+        #   locally available
+        self.stored_dbs = ecoinvent_interface.eidlstorage.stored_dbs
         self.stored_combobox.clear()
         self.stored_combobox.addItems(sorted(self.stored_dbs.keys()))
 
@@ -740,7 +752,7 @@ def report_failed_unarchive(self, file: str) -> None:
 class MainWorkerThread(ABThread):
     def __init__(self, downloader, parent=None):
         super().__init__(parent)
-        self.downloader = downloader
+        self.downloader: "ABEcoinventDownloader" = downloader
         self.forwast_url = (
             "https://lca-net.com/wp-content/uploads/forwast.bw2package.zip"
         )
@@ -784,16 +796,12 @@ def run_safely(self):
 
     def run_ecoinvent(self) -> None:
         """Run the ecoinvent downloader from start to finish."""
-        self.downloader.outdir = eidl.eidlstorage.eidl_dir
-        if self.downloader.check_stored():
-            import_signals.download_complete.emit()
-        else:
-            self.run_download()
+        archive_file = self.run_download()
 
         with tempfile.TemporaryDirectory() as tempdir:
             temp_dir = Path(tempdir)
             if not import_signals.cancel_sentinel:
-                self.run_extract(temp_dir)
+                self.run_extract(archive_file, temp_dir)
             if not import_signals.cancel_sentinel:
                 dataset_dir = temp_dir.joinpath("datasets")
                 self.run_import(dataset_dir)
@@ -823,17 +831,23 @@ def run_forwast(self) -> None:
             else:
                 self.delete_canceled_db()
 
-    def run_download(self) -> None:
+    def run_download(self) -> Path:
         """Use the connected ecoinvent downloader."""
-        self.downloader.download()
+        filepath = self.downloader.download()
         import_signals.download_complete.emit()
+        return filepath
 
-    def run_extract(self, temp_dir: Path) -> None:
+    def run_extract(self, archive_file: Path, temp_dir: Path) -> None:
         """Use the connected ecoinvent downloader to extract the downloaded
         7zip file.
         """
-        self.downloader.extract(target_dir=temp_dir)
-        import_signals.unarchive_finished.emit()
+        try:
+            self.downloader.extract(archive_file, temp_dir)
+        except Exception:
+            import_signals.cancel_sentinel = True
+            import_signals.unarchive_failed.emit(temp_dir)
+        else:
+            import_signals.unarchive_finished.emit()
 
     def run_extract_import(self) -> None:
         """Combine the extract and import steps when beginning from a selected
@@ -970,11 +984,20 @@ def __init__(self, parent=None):
         super().__init__(parent)
         self.wizard = parent
         self.complete = False
+        eco_settings = ecoinvent_interface.Settings()
         self.username_edit = QtWidgets.QLineEdit()
-        self.username_edit.setPlaceholderText("ecoinvent username")
+        if eco_settings.username:
+            self.username_edit.setText(eco_settings.username)
+        else:
+            self.username_edit.setPlaceholderText("ecoinvent username")
         self.password_edit = QtWidgets.QLineEdit()
-        self.password_edit.setPlaceholderText("ecoinvent password"),
         self.password_edit.setEchoMode(QtWidgets.QLineEdit.Password)
+        if eco_settings.password:
+            self.password_edit.setText(eco_settings.password)
+        else:
+            self.password_edit.setPlaceholderText("ecoinvent password")
+        self.save_creds = QtWidgets.QPushButton("Save Credentials")
+        self.save_creds.clicked.connect(self.save_credentials)
         self.login_button = QtWidgets.QPushButton("login")
         self.login_button.clicked.connect(self.login)
         self.password_edit.returnPressed.connect(self.login_button.click)
@@ -989,6 +1012,7 @@ def __init__(self, parent=None):
         box_layout.addWidget(self.password_edit)
         hlay = QtWidgets.QHBoxLayout()
         hlay.addWidget(self.login_button)
+        hlay.addWidget(self.save_creds)
         hlay.addStretch(1)
         box_layout.addLayout(hlay)
         box_layout.addWidget(self.success_label)
@@ -1018,6 +1042,13 @@ def login(self) -> None:
         self.login_thread.update(self.username, self.password)
         self.login_thread.start()
 
+    @Slot(name="SaveEiCredentials")
+    def save_credentials(self):
+        self.success_label.setText("Saving Credentials")
+        ecoinvent_interface.permanent_setting("username", self.username)
+        ecoinvent_interface.permanent_setting("password", self.password)
+        self.success_label.setText("Saved Credentials")
+
     @Slot(bool, name="handleLoginResponse")
     def login_response(self, success: bool) -> None:
         if not success:
@@ -1039,7 +1070,7 @@ def nextId(self):
 
 
 class LoginThread(QtCore.QThread):
-    def __init__(self, downloader, parent=None):
+    def __init__(self, downloader: "ABEcoinventDownloader", parent=None):
         super().__init__(parent)
         self.downloader = downloader
 
@@ -1048,13 +1079,31 @@ def update(self, username: str, password: str) -> None:
         self.downloader.password = password
 
     def run(self):
-        self.downloader.login()
+        error_message = None
+        try:
+            login_success, error_message = self.downloader.login()
+        except Exception as e:
+            log.error(str(e), exc_info=True)
+            import_signals.login_success.emit(False)
+            msg = str(e)
+            cs = ecoinvent_interface.CachedStorage()
+            if len(cs.catalogue) > 0:
+                msg += (
+                    "\n\nIf you work offline you can use your previously downloaded databases"
+                    + " via the archive option of the import wizard."
+                )
+            import_signals.connection_problem.emit(("Unexpected error", msg))
+        else:
+            import_signals.login_success.emit(login_success)
+        finally:
+            if error_message:
+                import_signals.connection_problem.emit(error_message)
 
 
 class EcoinventVersionPage(QtWidgets.QWizardPage):
     def __init__(self, parent=None):
         super().__init__(parent)
-        self.wizard = self.parent()
+        self.wizard: "DatabaseImportWizard" = self.parent()
         self.description_label = QtWidgets.QLabel(
             "Choose ecoinvent version and system model:"
         )
@@ -1065,37 +1114,33 @@ def __init__(self, parent=None):
             self.update_system_model_combobox
         )
         self.system_model_combobox = QtWidgets.QComboBox()
+        self.release_type_combobox = QtWidgets.QComboBox()
+        self.release_type_combobox.addItems(
+            [x.name for x in list(ecoinvent_interface.ReleaseType)]
+        )
 
         layout = QtWidgets.QGridLayout()
         layout.addWidget(self.description_label, 0, 0, 1, 3)
         layout.addWidget(QtWidgets.QLabel("Version: "), 1, 0)
         layout.addWidget(self.version_combobox, 1, 1, 1, 2)
         layout.addWidget(QtWidgets.QLabel("System model: "), 2, 0)
         layout.addWidget(self.system_model_combobox, 2, 1, 1, 2)
+        layout.addWidget(QtWidgets.QLabel("Release Type: "), 3, 0)
+        layout.addWidget(self.release_type_combobox, 3, 1, 1, 2)
         self.setLayout(layout)
 
     def initializePage(self):
-        if self.db_dict is None:
-            self.wizard.downloader.db_dict = (
-                self.wizard.downloader.get_available_files()
-            )
-            self.db_dict = self.wizard.downloader.db_dict
-        self.system_models = {
-            version: sorted(
-                {k[1] for k in self.db_dict.keys() if k[0] == version}, reverse=True
-            )
-            for version in sorted(
-                {k[0] for k in self.db_dict.keys() if k[0] in __ei_versions__},
-                reverse=True,
-            )
-        }
+        available_versions = self.wizard.downloader.list_versions()
+        shown_versions = set(
+            [version for version in available_versions if version in __ei_versions__]
+        )
         # Catch for incorrect 'universal' key presence
         # (introduced in version 3.6 of ecoinvent)
-        if "universal" in self.system_models:
-            del self.system_models["universal"]
+        if "universal" in shown_versions:
+            shown_versions.remove("universal")
         self.version_combobox.clear()
         self.system_model_combobox.clear()
-        versions = sort_semantic_versions(self.system_models.keys())
+        versions = sort_semantic_versions(shown_versions)
         self.version_combobox.addItems(versions)
         if bool(self.version_combobox.count()):
             # Adding the items will cause system_model_combobox to update
@@ -1120,7 +1165,9 @@ def update_system_model_combobox(self, version: str) -> None:
         different ecoinvent version.
         """
         self.system_model_combobox.clear()
-        self.system_model_combobox.addItems(self.system_models[version])
+        items = self.wizard.downloader.list_system_models(version)
+        items = sorted(items, reverse=True)
+        self.system_model_combobox.addItems(items)
 
 
 class LocalDatabaseImportPage(QtWidgets.QWizardPage):
@@ -1316,27 +1363,116 @@ class ImportSignals(QtCore.QObject):
 import_signals = ImportSignals()
 
 
-class ABEcoinventDownloader(eidl.EcoinventDownloader):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.extraction_process = None
+# TODO: reimplement downloader using ecoinvent_interface
+class ABEcoinventDownloader(object):
+    def __init__(
+        self,
+        version=None,
+        system_model=None,
+        release_type: typing.Optional[ecoinvent_interface.ReleaseType] = None,
+    ):
+        self.version = version
+        self.system_model = system_model
+        self._release_type = release_type
+        self._settings = ecoinvent_interface.Settings()
+        self._release = ecoinvent_interface.EcoinventRelease(self._settings)
+
+    def update_ecoinvent_release(self):
+        self._release = ecoinvent_interface.EcoinventRelease(self._settings)
 
-    def login_success(self, success):
-        import_signals.login_success.emit(success)
+    @property
+    def username(self):
+        return self._settings.username
 
-    def extract(self, target_dir):
-        """Override extract method to redirect the stdout to dev null."""
-        code = super().extract(target_dir=target_dir, stdout=subprocess.DEVNULL)
-        if code != 0:
-            # The archive was corrupted in some way.
-            import_signals.cancel_sentinel = True
-            import_signals.unarchive_failed.emit(self.out_path)
-
-    def handle_connection_timeout(self):
-        msg = "The request timed out, please check your internet connection!"
-        if eidl.eidlstorage.stored_dbs:
-            msg += (
-                "\n\nIf you work offline you can use your previously downloaded databases"
-                + " via the archive option of the import wizard."
+    @username.setter
+    def username(self, value):
+        self._settings.username = value
+        self.update_ecoinvent_release()
+
+    @property
+    def password(self):
+        return self._settings.password
+
+    @password.setter
+    def password(self, value):
+        self._settings.password = value
+        self.update_ecoinvent_release()
+
+    @property
+    def release_type(self):
+        return self._release_type
+
+    @release_type.setter
+    def release_type(self, value: typing.Union[str, ecoinvent_interface.ReleaseType]):
+        if isinstance(value, ecoinvent_interface.ReleaseType):
+            self._release_type = value
+            return
+
+        if isinstance(value, str):
+            self._release_type = ecoinvent_interface.ReleaseType[value]
+            return
+
+        raise ValueError("invalid value provided for release_type")
+
+    def login(self) -> (bool, typing.Optional[typing.Tuple[str, str]]):
+        release = ecoinvent_interface.EcoinventRelease(self._settings)
+        error_message = None
+        try:
+            release.login()
+            login_success = True
+        except (
+            requests.ConnectTimeout,
+            requests.ReadTimeout,
+            requests.ConnectionError,
+        ) as e:
+            login_success = False
+            error_message = (
+                "Connection Problem",
+                "The request timed out, please check your internet connection!",
             )
-        import_signals.connection_problem.emit(("Connection problem", msg))
+        except requests.exceptions.HTTPError as e:
+            login_success = False
+            error_message = None
+            if e.response.status_code != 401:
+                log.error(
+                    "Unexpected status code (%d) received when trying to list ecoinvent_versions, response: %s",
+                    e.response.status_code,
+                    e.response.text,
+                )
+                error_message = (
+                    "Unexpected Problem",
+                    "An unexpected error occurred, please try again status code %d"
+                    % e.response.status_code,
+                )
+
+        return login_success, error_message
+
+    @lru_cache(maxsize=1)
+    def list_versions(self):
+        return self._release.list_versions()
+
+    @lru_cache(maxsize=100)
+    def list_system_models(self, version: str):
+        return self._release.list_system_models(version)
+
+    def download(self) -> Path:
+        return self._release.get_release(
+            version=self.version,
+            system_model=self.system_model,
+            release_type=self.release_type,
+            extract=False,
+        )
+
+    @staticmethod
+    def extract(filepath: Path, out_dir: Path = None):
+        """
+        Extract archive
+        """
+        if filepath.suffix.lower() == ".7z":
+            with py7zr.SevenZipFile(filepath, "r") as archive:
+                directory = out_dir or (filepath.parent / filepath.stem)
+                if directory.exists():
+                    shutil.rmtree(directory)
+                archive.extractall(path=directory)
+        else:
+            raise ValueError("Unsupported archive format")