From 373f867df86d7656d18d492d1a84895361ccf2ff Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Thu, 25 Nov 2021 10:43:02 +0000 Subject: [PATCH 01/11] Add new config option for master install path, and compare versions on startup --- .isort.cfg | 2 +- config.py | 3 +++ default_config.ini | 3 +++ docs/configuration.rst | 7 +++++++ pepys_import/core/store/data_store.py | 21 +++++++++++++++++++++ pepys_import/utils/data_store_utils.py | 16 ++++++++++++++++ requirements.txt | 3 ++- 7 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index 1295ffb1d..88d9682bc 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -4,4 +4,4 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true line_length = 100 -known_third_party =alembic,dateutil,flask,flask_cachebuster,flask_cors,freezegun,geoalchemy2,geopy,halo,iterfzf,loguru,lxml,pexpect,pg8000,pint,prompt_toolkit,psycopg2,pyfiglet,pygments,pytest,setuptools,shapely,sqlalchemy,sqlalchemy_utils,tabulate,testing,tqdm,twisted,waitress \ No newline at end of file +known_third_party =alembic,dateutil,flask,flask_cachebuster,flask_cors,freezegun,geoalchemy2,geopy,halo,iterfzf,loguru,lxml,packaging,pexpect,pg8000,pint,prompt_toolkit,psycopg2,pyfiglet,pygments,pytest,setuptools,shapely,sqlalchemy,sqlalchemy_utils,tabulate,testing,tqdm,twisted,waitress \ No newline at end of file diff --git a/config.py b/config.py index 59c0c6c19..ab25170c0 100644 --- a/config.py +++ b/config.py @@ -66,3 +66,6 @@ LOCAL_PARSERS = config.get("local", "parsers", fallback="") LOCAL_BASIC_TESTS = config.get("local", "basic_tests", fallback="") LOCAL_ENHANCED_TESTS = config.get("local", "enhanced_tests", fallback="") + +# Fetch network section +NETWORK_MASTER_INSTALL_PATH = config.get("network", "master_install_path", fallback="") diff --git a/default_config.ini b/default_config.ini index b1e701830..70fd313e7 100644 --- a/default_config.ini +++ b/default_config.ini @@ -13,3 +13,6 @@ path = archive parsers = basic_tests = enhanced_tests = +[network] +master_install_path = + diff --git a/docs/configuration.rst b/docs/configuration.rst index fc8841c80..3f6376f82 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -57,6 +57,13 @@ The specific variables are: - :code:`basic_tests`: Path to a folder containing custom basic validation tests to be loaded by pepys-import (default: none) - :code:`enhanced_tests`: Path to a folder containing custom enhanced validation tests to be loaded by pepys-import (default: none) +:code:`[network]` section +######################### +These settings control which paths Pepys looks for on the network. The specific variables are: + + - :code:`master_install_location`: Path to a folder containing the 'master' install of Pepys. The current Pepys version will be checked against + this install, and an error will be given if the master version is newer. + Encryption ########## Various fields can be specified in an encrypted form (see notes above). To do this, diff --git a/pepys_import/core/store/data_store.py b/pepys_import/core/store/data_store.py index fc639178a..a80bda7d4 100644 --- a/pepys_import/core/store/data_store.py +++ b/pepys_import/core/store/data_store.py @@ -9,6 +9,7 @@ import pint import sqlalchemy +from packaging import version from sqlalchemy import create_engine, inspect from sqlalchemy.event import listen from sqlalchemy.exc import ArgumentError, OperationalError @@ -17,6 +18,8 @@ from sqlalchemy.sql.expression import text from sqlalchemy_utils import dependent_objects, get_referencing_foreign_keys, merge_references +import config +import pepys_import from paths import MIGRATIONS_DIRECTORY, PEPYS_IMPORT_DIRECTORY from pepys_admin.utils import read_latest_revisions_file from pepys_import import __build_timestamp__, __version__ @@ -36,6 +39,7 @@ create_stored_procedures_for_postgres, import_from_csv, lowercase_or_none, + read_version_from_pepys_install, ) from pepys_import.utils.sqlite_utils import load_spatialite, set_sqlite_foreign_keys_on from pepys_import.utils.value_transforming_utils import format_datetime @@ -224,6 +228,8 @@ def __init__( # of the same length makes things prettier print("-" * 78) + self.check_network_version() + def initialise(self): """Create schemas for the database""" if self.db_type == "sqlite": @@ -269,6 +275,21 @@ def session_scope(self): finally: self.session.close() + def check_network_version(self): + if config.NETWORK_MASTER_INSTALL_PATH != "": + network_version = read_version_from_pepys_install(config.NETWORK_MASTER_INSTALL_PATH) + current_version = pepys_import.__version__ + + if network_version is None: + # We've already given a warning that we can't read the network version, so just continue + return + + if version.parse(network_version) > version.parse(current_version): + print( + "Your version of Pepys is out of date. Please run the Update Pepys program from the Start Menu." + ) + sys.exit(1) + ############################################################# # Other DataStore Methods diff --git a/pepys_import/utils/data_store_utils.py b/pepys_import/utils/data_store_utils.py index 2753b2455..ac9165715 100644 --- a/pepys_import/utils/data_store_utils.py +++ b/pepys_import/utils/data_store_utils.py @@ -433,3 +433,19 @@ def convert_objects_to_ids(items, table_obj): else: value = items return value + + +def read_version_from_pepys_install(path): + init_path = os.path.join(path, "pepys_import", "__init__.py") + + try: + with open(init_path, "r") as f: + for line in f: + if "__version__" in line: + splitted = line.split("=") + # Remove whitespace, double-quotes and single-quotes from either end + version = splitted[1].strip().strip('"').strip("'") + return version + except Exception: + print(f"WARNING: Cannot read Pepys version from network master install at {path}") + return None diff --git a/requirements.txt b/requirements.txt index 727331097..3d65a9d4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,5 @@ Flask Flask-CacheBuster flask-cors waitress -lxml \ No newline at end of file +lxml +packaging \ No newline at end of file From 71b2edeb3c6830877b8a93f3405bbeb97f20e7ba Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 29 Nov 2021 16:59:38 +0000 Subject: [PATCH 02/11] Add scripts to upgrade Pepys, incl Start Menu shortcuts --- bin/install_pepys.ps1 | 2 ++ bin/upgrade_pepys.bat | 13 +++++++++++++ bin/upgrade_pepys.ps1 | 14 ++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 bin/upgrade_pepys.bat create mode 100644 bin/upgrade_pepys.ps1 diff --git a/bin/install_pepys.ps1 b/bin/install_pepys.ps1 index b0c6a531a..3887d4961 100644 --- a/bin/install_pepys.ps1 +++ b/bin/install_pepys.ps1 @@ -92,6 +92,8 @@ try { # Pepys Viewer shortcuts Make-Shortcut -ShortcutLocation ($startmenu_location + "Pepys\Pepys Viewer.lnk") -TargetPath ".\pepys_viewer.bat" -Icon $icon_string Make-Shortcut -ShortcutLocation ($startmenu_location + "Pepys\Pepys Viewer (training mode).lnk") -TargetPath ".\pepys_viewer_training.bat" -Icon $icon_string + + Make-Shortcut -ShortcutLocation ($startmenu_location + "Pepys\Upgrade Pepys.lnk") -TargetPath ".\upgrade_pepys.bat" -Icon $icon_string } catch { Write-Output $_ diff --git a/bin/upgrade_pepys.bat b/bin/upgrade_pepys.bat new file mode 100644 index 000000000..7e125de52 --- /dev/null +++ b/bin/upgrade_pepys.bat @@ -0,0 +1,13 @@ +@ECHO off +CALL set_title.bat Upgrade Pepys +CALL set_paths.bat + +powershell.exe -executionpolicy remotesigned -File upgrade_pepys.ps1 +IF ERRORLEVEL 1 GOTO :ERROR +ECHO Pepys Upgrade completed +PAUSE + +GOTO :eof + +:ERROR +ECHO Error running upgrade_pepys.ps1 \ No newline at end of file diff --git a/bin/upgrade_pepys.ps1 b/bin/upgrade_pepys.ps1 new file mode 100644 index 000000000..a948f00c1 --- /dev/null +++ b/bin/upgrade_pepys.ps1 @@ -0,0 +1,14 @@ +$network_path = python -c "import config; print(config.NETWORK_MASTER_INSTALL_PATH)" + +Write-Output "Copying new version of Pepys from $network_path" + +try { + Copy-Item -Path $network_path -Destination "..\" -Recurse +} +catch { + Write-Output $_ + Write-Output "ERROR: Could not copy new Pepys version" + Exit 1 +} + +Write-Output "Copy completed" \ No newline at end of file From 3d7bf276a0e396f8850ba8e7eb43fcfacbd50535 Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 29 Nov 2021 17:23:45 +0000 Subject: [PATCH 03/11] Copy to correct place, with Verbose and Force flags, ignoring .git directory --- bin/upgrade_pepys.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/upgrade_pepys.ps1 b/bin/upgrade_pepys.ps1 index a948f00c1..dcc2465b6 100644 --- a/bin/upgrade_pepys.ps1 +++ b/bin/upgrade_pepys.ps1 @@ -1,9 +1,14 @@ $network_path = python -c "import config; print(config.NETWORK_MASTER_INSTALL_PATH)" +if ($network_path -eq "") { + Write-Output "Network path not set in config file, cannot copy." + Exit 1 +} + Write-Output "Copying new version of Pepys from $network_path" try { - Copy-Item -Path $network_path -Destination "..\" -Recurse + Copy-Item -Path "$network_path\*" -Destination "..\" -Recurse -Force -Verbose -Exclude ".git" } catch { Write-Output $_ From ac0eada83f44ee11a7440ed6c6243fa374882c40 Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Mon, 29 Nov 2021 17:26:35 +0000 Subject: [PATCH 04/11] Add comments --- bin/upgrade_pepys.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/upgrade_pepys.ps1 b/bin/upgrade_pepys.ps1 index dcc2465b6..cb94e88e0 100644 --- a/bin/upgrade_pepys.ps1 +++ b/bin/upgrade_pepys.ps1 @@ -1,3 +1,5 @@ +# Get the path to the network master install from the Pepys config file +# To make this as reliable as possible, we use Pepys own config parsing code $network_path = python -c "import config; print(config.NETWORK_MASTER_INSTALL_PATH)" if ($network_path -eq "") { From 7203bbddf7324b8d3f06a46bfda737fe3bcd91ea Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Thu, 2 Dec 2021 20:42:52 +0000 Subject: [PATCH 05/11] Delete folder first --- bin/upgrade_pepys.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/upgrade_pepys.ps1 b/bin/upgrade_pepys.ps1 index cb94e88e0..fe516c74e 100644 --- a/bin/upgrade_pepys.ps1 +++ b/bin/upgrade_pepys.ps1 @@ -9,8 +9,11 @@ if ($network_path -eq "") { Write-Output "Copying new version of Pepys from $network_path" +Set-Location -Path .. + try { - Copy-Item -Path "$network_path\*" -Destination "..\" -Recurse -Force -Verbose -Exclude ".git" + Remove-Item -Path * -Recurse -Force + Copy-Item -Path "$network_path\*" -Destination "." -Recurse -Force -Verbose -Exclude ".git" } catch { Write-Output $_ From 3e74238a63f0236be63ddc80811b6bd994ee907e Mon Sep 17 00:00:00 2001 From: Robin Wilson Date: Thu, 2 Dec 2021 21:10:27 +0000 Subject: [PATCH 06/11] Change dirs and delete in parts to get around Powershell keeping hold of reference to directory --- bin/upgrade_pepys.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bin/upgrade_pepys.ps1 b/bin/upgrade_pepys.ps1 index fe516c74e..b50dcab22 100644 --- a/bin/upgrade_pepys.ps1 +++ b/bin/upgrade_pepys.ps1 @@ -7,12 +7,20 @@ if ($network_path -eq "") { Exit 1 } -Write-Output "Copying new version of Pepys from $network_path" -Set-Location -Path .. try { - Remove-Item -Path * -Recurse -Force + Write-Output "Deleting old version of Pepys" + Set-Location -Path .. + # Delete all except the bin directory, as that directory is still kept open by the Powershell/CMD process + Remove-Item -Path * -Recurse -Force -Exclude bin -Verbose + # Move to bin and delete all the contents, but not the folder (it's fine to keep the folder, as we're + # always going to have a bin directory) + Set-Location -Path bin + Remove-Item -Path * + # Change back to the main directory + Set-Location -Path .. + Write-Output "Copying new version of Pepys from $network_path" Copy-Item -Path "$network_path\*" -Destination "." -Recurse -Force -Verbose -Exclude ".git" } catch { From fbc82681365203b80dc12eba555253f4ec87abc7 Mon Sep 17 00:00:00 2001 From: Ian Mayo Date: Mon, 6 Dec 2021 10:50:06 +0000 Subject: [PATCH 07/11] Minor language change, correct shortcut name. --- pepys_import/core/store/data_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pepys_import/core/store/data_store.py b/pepys_import/core/store/data_store.py index a80bda7d4..daebe90fd 100644 --- a/pepys_import/core/store/data_store.py +++ b/pepys_import/core/store/data_store.py @@ -286,7 +286,7 @@ def check_network_version(self): if version.parse(network_version) > version.parse(current_version): print( - "Your version of Pepys is out of date. Please run the Update Pepys program from the Start Menu." + "Your local installation of Pepys is older than than the network master. Please run `Upgrade Pepys` from the Start Menu." ) sys.exit(1) From 26d15abbda0ce89f9d8abee3f5634c14cfc32da4 Mon Sep 17 00:00:00 2001 From: Ian Mayo Date: Mon, 6 Dec 2021 12:53:03 +0000 Subject: [PATCH 08/11] Remove whitespace, to keep linter happy --- tests/gui_tests/gui_test_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/gui_tests/gui_test_utils.py b/tests/gui_tests/gui_test_utils.py index dd43910b0..f17f00d07 100644 --- a/tests/gui_tests/gui_test_utils.py +++ b/tests/gui_tests/gui_test_utils.py @@ -66,7 +66,6 @@ def run_gui(ds, keys=None, print_output=False): return "\n".join(screen.display) - else: def run_gui(): From 1575a4a8437093579f6024774704c08885f50124 Mon Sep 17 00:00:00 2001 From: Ian Mayo Date: Mon, 6 Dec 2021 13:41:28 +0000 Subject: [PATCH 09/11] Mark method as static, to please linter --- pepys_import/core/store/data_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pepys_import/core/store/data_store.py b/pepys_import/core/store/data_store.py index daebe90fd..a97cc4493 100644 --- a/pepys_import/core/store/data_store.py +++ b/pepys_import/core/store/data_store.py @@ -275,6 +275,7 @@ def session_scope(self): finally: self.session.close() + @staticmethod def check_network_version(self): if config.NETWORK_MASTER_INSTALL_PATH != "": network_version = read_version_from_pepys_install(config.NETWORK_MASTER_INSTALL_PATH) From e9491ae8a95af8cb5bb28f481841ae8e3c48bfa0 Mon Sep 17 00:00:00 2001 From: Ian Mayo Date: Mon, 6 Dec 2021 13:41:41 +0000 Subject: [PATCH 10/11] Attempt to overcome static linter warning --- pepys_import/utils/data_store_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pepys_import/utils/data_store_utils.py b/pepys_import/utils/data_store_utils.py index ac9165715..2b97e52f5 100644 --- a/pepys_import/utils/data_store_utils.py +++ b/pepys_import/utils/data_store_utils.py @@ -446,6 +446,7 @@ def read_version_from_pepys_install(path): # Remove whitespace, double-quotes and single-quotes from either end version = splitted[1].strip().strip('"').strip("'") return version + return None except Exception: print(f"WARNING: Cannot read Pepys version from network master install at {path}") return None From b49184de1b4b1e3e9765845a649052b9c2201681 Mon Sep 17 00:00:00 2001 From: Ian Mayo Date: Mon, 6 Dec 2021 13:47:23 +0000 Subject: [PATCH 11/11] remove failed fix --- pepys_import/core/store/data_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pepys_import/core/store/data_store.py b/pepys_import/core/store/data_store.py index a97cc4493..daebe90fd 100644 --- a/pepys_import/core/store/data_store.py +++ b/pepys_import/core/store/data_store.py @@ -275,7 +275,6 @@ def session_scope(self): finally: self.session.close() - @staticmethod def check_network_version(self): if config.NETWORK_MASTER_INSTALL_PATH != "": network_version = read_version_from_pepys_install(config.NETWORK_MASTER_INSTALL_PATH)