From 3b9ad1422f5846981713ae650da4d7c7b2ff9cec Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:50:52 -0800 Subject: [PATCH 1/3] Merge pull request #2822 from schandrika/fix_sqlhistorian_metadata_cache fix for issue #2821 --- volttron/platform/dbutils/mysqlfuncts.py | 2 +- volttron/platform/dbutils/postgresqlfuncts.py | 2 +- volttron/platform/dbutils/redshiftfuncts.py | 2 +- volttron/platform/dbutils/sqlitefuncts.py | 2 +- .../platform/dbutils/test_mysqlfuncts.py | 17 +++++++++++++++++ .../dbutils/test_postgresql_timescaledb.py | 16 ++++++++++++++++ .../platform/dbutils/test_postgresqlfuncts.py | 17 +++++++++++++++++ .../platform/dbutils/test_sqlitefuncts.py | 16 ++++++++++++++++ 8 files changed, 70 insertions(+), 4 deletions(-) diff --git a/volttron/platform/dbutils/mysqlfuncts.py b/volttron/platform/dbutils/mysqlfuncts.py index 4bbb754b33..50b4729495 100644 --- a/volttron/platform/dbutils/mysqlfuncts.py +++ b/volttron/platform/dbutils/mysqlfuncts.py @@ -414,7 +414,7 @@ def get_topic_meta_map(self): _log.debug("loading metadata from db") topic_meta_map = dict() for id, meta in rows: - topic_meta_map[id] = jsonapi.loads(meta) + topic_meta_map[id] = jsonapi.loads(meta) if meta else None return topic_meta_map def get_topic_map(self): diff --git a/volttron/platform/dbutils/postgresqlfuncts.py b/volttron/platform/dbutils/postgresqlfuncts.py index 0983611388..8e74934371 100644 --- a/volttron/platform/dbutils/postgresqlfuncts.py +++ b/volttron/platform/dbutils/postgresqlfuncts.py @@ -350,7 +350,7 @@ def get_topic_meta_map(self): 'SELECT topic_id, metadata ' 'FROM {}').format(Identifier(self.meta_table)) rows = self.select(query) - meta_map = {tid: jsonapi.loads(meta) for tid, meta in rows} + meta_map = {tid: jsonapi.loads(meta) if meta else None for tid, meta in rows} return meta_map def get_agg_topics(self): diff --git a/volttron/platform/dbutils/redshiftfuncts.py b/volttron/platform/dbutils/redshiftfuncts.py index e7c51dca1a..934b9015ae 100644 --- a/volttron/platform/dbutils/redshiftfuncts.py +++ b/volttron/platform/dbutils/redshiftfuncts.py @@ -266,7 +266,7 @@ def get_topic_meta_map(self): 'SELECT topic_id, metadata ' 'FROM {}').format(Identifier(self.meta_table)) rows = self.select(query) - meta_map = {tid: jsonapi.loads(meta) for tid, meta in rows} + meta_map = {tid: jsonapi.loads(meta) if meta else None for tid, meta in rows} return meta_map def get_agg_topics(self): diff --git a/volttron/platform/dbutils/sqlitefuncts.py b/volttron/platform/dbutils/sqlitefuncts.py index 1a48aa7c45..5c525a63cd 100644 --- a/volttron/platform/dbutils/sqlitefuncts.py +++ b/volttron/platform/dbutils/sqlitefuncts.py @@ -384,7 +384,7 @@ def get_topic_meta_map(self): _log.debug("loading metadata from db") topic_meta_map = dict() for id, meta in rows: - topic_meta_map[id] = jsonapi.loads(meta) + topic_meta_map[id] = jsonapi.loads(meta) if meta else None return topic_meta_map def get_agg_topics(self): diff --git a/volttrontesting/platform/dbutils/test_mysqlfuncts.py b/volttrontesting/platform/dbutils/test_mysqlfuncts.py index 95d5d6e0ed..13cad99927 100644 --- a/volttrontesting/platform/dbutils/test_mysqlfuncts.py +++ b/volttrontesting/platform/dbutils/test_mysqlfuncts.py @@ -263,6 +263,23 @@ def test_get_topic_map_should_succeed(get_container_func): assert actual == expected +def test_get_topic_meta_map_should_succeed(get_container_func): + container, sqlfuncts, connection_port, historian_version = get_container_func + if historian_version == "<4.0.0": + pytest.skip("method applied only to version >=4.0.0") + else: + query = """ + INSERT INTO topics (topic_name) + VALUES ('football'); + INSERT INTO topics (topic_name, metadata) + VALUES ('baseball', '{"metadata":"value"}'); + """ + seed_database(container, query) + expected = {1: None, 2: {"metadata": "value"}} + actual = sqlfuncts.get_topic_meta_map() + assert actual == expected + + # fails for image:mysql:8.0.25 historian schema version >=4.0.0 def test_get_agg_topic_map_should_return_dict(get_container_func): container, sqlfuncts, connection_port, historian_version = get_container_func diff --git a/volttrontesting/platform/dbutils/test_postgresql_timescaledb.py b/volttrontesting/platform/dbutils/test_postgresql_timescaledb.py index 62baff85f6..0904d50c01 100644 --- a/volttrontesting/platform/dbutils/test_postgresql_timescaledb.py +++ b/volttrontesting/platform/dbutils/test_postgresql_timescaledb.py @@ -331,6 +331,22 @@ def test_get_topic_map_should_return_maps(get_container_func): assert actual == expected +def test_get_topic_meta_map_should_return_maps(get_container_func): + container, sqlfuncts, connection_port, historian_version = get_container_func + if historian_version == "<4.0.0": + pytest.skip("method applied only to version >=4.0.0") + else: + query = """ + INSERT INTO topics (topic_name) + VALUES ('football'); + INSERT INTO topics (topic_name, metadata) + VALUES ('baseball', '{"meta":"value"}'); + """ + seed_database(container, query) + expected = {1: None, 2: {"meta": "value"}} + actual = sqlfuncts.get_topic_meta_map() + assert actual == expected + def test_get_agg_topics_should_return_list(get_container_func): container, postgresqlfuncts, port_on_host, historian_version = get_container_func topic = "some_agg_topic" diff --git a/volttrontesting/platform/dbutils/test_postgresqlfuncts.py b/volttrontesting/platform/dbutils/test_postgresqlfuncts.py index 26fa605c34..172f29761a 100644 --- a/volttrontesting/platform/dbutils/test_postgresqlfuncts.py +++ b/volttrontesting/platform/dbutils/test_postgresqlfuncts.py @@ -338,6 +338,23 @@ def test_get_topic_map_should_return_maps(get_container_func): assert actual == expected +def test_get_topic_meta_map_should_return_maps(get_container_func): + container, sqlfuncts, connection_port, historian_version = get_container_func + if historian_version == "<4.0.0": + pytest.skip("method applied only to version >=4.0.0") + else: + query = """ + INSERT INTO topics (topic_name) + VALUES ('football'); + INSERT INTO topics (topic_name, metadata) + VALUES ('baseball', '{"meta":"value"}'); + """ + seed_database(container, query) + expected = {1: None, 2: {"meta": "value"}} + actual = sqlfuncts.get_topic_meta_map() + assert actual == expected + + def test_get_agg_topics_should_return_list(get_container_func): container, sqlfuncts, connection_port, historian_version = get_container_func diff --git a/volttrontesting/platform/dbutils/test_sqlitefuncts.py b/volttrontesting/platform/dbutils/test_sqlitefuncts.py index acd1d49a4c..490df0fb79 100644 --- a/volttrontesting/platform/dbutils/test_sqlitefuncts.py +++ b/volttrontesting/platform/dbutils/test_sqlitefuncts.py @@ -276,6 +276,22 @@ def test_get_topic_map(get_sqlitefuncts): assert actual_topic_map == expected_topic_map +@pytest.mark.sqlitefuncts +@pytest.mark.dbutils +def test_get_topic_meta_map(get_sqlitefuncts): + sqlitefuncts, historian_version = get_sqlitefuncts + if historian_version == "<4.0.0": + pytest.skip("method applied only to version >=4.0.0") + else: + query = "INSERT INTO topics (topic_name) values ('football');" \ + "INSERT INTO topics (topic_name, metadata) values ('netball', '{\"meta\":\"value\"}');" + query_db(query) + expected_topic_map = {1: None, 2: {"meta": "value"}} + + actual_topic_meta_map = sqlitefuncts.get_topic_meta_map() + + assert actual_topic_meta_map == expected_topic_map + @pytest.mark.sqlitefuncts @pytest.mark.dbutils def test_get_agg_topics(get_sqlitefuncts): From e9c17fac976bcd97466271ba7f353888c336fc27 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:47:34 -0800 Subject: [PATCH 2/3] Update __init__.py Update to version 8.1.2 --- volttron/platform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index 2f33379f47..6eefc75849 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -49,7 +49,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '8.1.1' +__version__ = '8.1.2' _log = logging.getLogger(__name__) From c97e1b08c408dc87e503f728575a7ec84de649e6 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:48:12 -0800 Subject: [PATCH 3/3] Update conf.py Update to version 8.1.2 --- docs/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 70d3290f4a..438df38645 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,9 +43,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '8.1.1' +version = '8.1.2' # The full version, including alpha/beta/rc tags -release = '8.1.1' +release = '8.1.2' # -- General configuration ---------------------------------------------------