Skip to content

Commit

Permalink
Releases/9.0.1 (#3171)
Browse files Browse the repository at this point in the history
* Update readthedocs requirements.txt

* Update conf.py

* Update requirements_demo.txt

Add missing pandas requirement for demo

* work around for issue #3154

* Fix for security issue #3168 (#3169)

* Fix for security issue #3168

* handling clean up errors in test

* testing group commands in different test module

* moved group and role test to different module

* moved group and role test to different module

* Added a cache for agent names since platform start

* Fixes process overload from file events

* fixed issue with variable definition.

* Remove PersistentDict from web-user.json file.

* Update admin_endpoints.py

Handle behavior of removing PersistentDict

* Update version to 9.0.1

---------

Co-authored-by: Chandrika Sivaramakrishnan <[email protected]>
Co-authored-by: Chandrika <[email protected]>
Co-authored-by: Andrew Rodgers <[email protected]>
  • Loading branch information
4 people authored May 8, 2024
1 parent 5b45014 commit 7b5339b
Show file tree
Hide file tree
Showing 13 changed files with 437 additions and 43 deletions.
4 changes: 2 additions & 2 deletions docs/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ def __getattr__(cls, name):
author = 'The VOLTTRON Community'

# The short X.Y version
version = '9.0'
version = '9.0.1'
# The full version, including alpha/beta/rc tags
release = '9.0'
release = '9.0.1'

# -- General configuration ---------------------------------------------------

Expand Down
1 change: 1 addition & 0 deletions services/core/IEEE_2030_5/requirements_demo.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ nicegui
requests
xsdata>=23.8
blinker
pandas
2 changes: 1 addition & 1 deletion volttron/platform/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
from urllib.parse import urlparse

from ..utils.frozendict import FrozenDict
__version__ = '9.0rc0'
__version__ = '9.0.1'

_log = logging.getLogger(__name__)

Expand Down
4 changes: 4 additions & 0 deletions volttron/platform/aip.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ def __init__(self, env, **kwargs):
if self.message_bus == 'rmq':
self.rmq_mgmt = RabbitMQMgmt()
self.instance_name = get_platform_instance_name()
self.agent_uuid_name_map = {}

def add_agent_user_group(self):
user = pwd.getpwuid(os.getuid())
Expand Down Expand Up @@ -682,11 +683,14 @@ def remove_agent(self, agent_uuid, remove_auth=True):
self.remove_agent_user(volttron_agent_user)

def agent_name(self, agent_uuid):
if cached_name := self.agent_uuid_name_map.get(agent_uuid):
return cached_name
agent_path = os.path.join(self.install_dir, agent_uuid)
for agent_name in os.listdir(agent_path):
dist_info = os.path.join(
agent_path, agent_name, agent_name + '.dist-info')
if os.path.exists(dist_info):
self.agent_uuid_name_map[agent_uuid] = agent_name
return agent_name
raise KeyError(agent_uuid)

Expand Down
3 changes: 0 additions & 3 deletions volttron/platform/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,9 +435,6 @@ def issue(self, topic, frames, extra=None):
# return result

def handle_subsystem(self, frames, user_id):
_log.debug(
f"Handling subsystem with frames: {frames} user_id: {user_id}")

subsystem = frames[5]
if subsystem == 'quit':
sender = frames[0]
Expand Down
6 changes: 2 additions & 4 deletions volttron/platform/vip/agent/subsystems/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ def update_rpc_method_capabilities(self):
"""
rpc_method_authorizations = {}
rpc_methods = self.get_rpc_exports()
updated_rpc_authorizations = None
for method in rpc_methods:
if len(method.split(".")) > 1:
pass
Expand Down Expand Up @@ -295,9 +296,7 @@ def update_rpc_method_capabilities(self):
_log.info(
f"Skipping updating rpc auth capabilities for agent "
f"{self._core().identity} connecting to remote address: {self._core().address} ")
updated_rpc_authorizations = None
except gevent.timeout.Timeout:
updated_rpc_authorizations = None
_log.warning(f"update_id_rpc_authorization rpc call timed out for {self._core().identity} {rpc_method_authorizations}")
except MethodNotFound:
_log.warning("update_id_rpc_authorization method is missing from "
Expand All @@ -306,7 +305,6 @@ def update_rpc_method_capabilities(self):
"dynamic RPC authorizations.")
return
except Exception as e:
updated_rpc_authorizations = None
_log.exception(f"Exception when calling rpc method update_id_rpc_authorizations for identity: "
f"{self._core().identity} Exception:{e}")
if updated_rpc_authorizations is None:
Expand All @@ -318,7 +316,7 @@ def update_rpc_method_capabilities(self):
f"the identity of the agent"
)
return
if rpc_method_authorizations != updated_rpc_authorizations:
if rpc_method_authorizations != updated_rpc_authorizations and updated_rpc_authorizations is not None:
for method in updated_rpc_authorizations:
self.set_rpc_authorizations(
method, updated_rpc_authorizations[method]
Expand Down
4 changes: 2 additions & 2 deletions volttron/platform/vip/agent/subsystems/rpc.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,8 @@ def _iterate_exports(self):
for method_name in self._exports:
method = self._exports[method_name]
caps = annotations(method, set, "rpc.allow_capabilities")
# if caps:
# self._exports[method_name] = self._add_auth_check(method, caps)
if caps:
self._exports[method_name] = self._add_auth_check(method, caps)

def _add_auth_check(self, method, required_caps):
"""
Expand Down
15 changes: 11 additions & 4 deletions volttron/platform/web/admin_endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
from volttron.platform import get_home
from volttron.platform import jsonapi
from volttron.utils import VolttronHomeFileReloader
from volttron.utils.persistance import PersistentDict


_log = logging.getLogger(__name__)
Expand Down Expand Up @@ -84,7 +83,7 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None)
else:
self._ssl_public_key = None

self._userdict = None
self._userdict = {}
self.reload_userdict()

self._observer = Observer()
Expand All @@ -96,7 +95,14 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None)

def reload_userdict(self):
webuserpath = os.path.join(get_home(), 'web-users.json')
self._userdict = PersistentDict(webuserpath, format="json")
if os.path.exists(webuserpath):
with open(webuserpath) as fp:
try:
self._userdict = jsonapi.loads(fp.read())
except json.decoder.JSONDecodeError:
self._userdict = {}
# Keep same behavior as with PersistentDict
raise ValueError("File not in a supported format")

def get_routes(self):
"""
Expand Down Expand Up @@ -339,4 +345,5 @@ def add_user(self, username, unencrypted_pw, groups=None, overwrite=False):
groups=groups
)

self._userdict.sync()
with open(os.path.join(get_home(), 'web-users.json'), 'w') as fp:
fp.write(jsonapi.dumps(self._userdict, indent=2))
4 changes: 2 additions & 2 deletions volttron/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ def __init__(self, filetowatch, callback):
_log.debug("patterns is {}".format([get_home() + '/' + filetowatch]))
self._callback = callback

def on_any_event(self, event):
def on_closed(self, event):
_log.debug("Calling callback on event {}. Calling {}".format(event, self._callback))
try:
self._callback()
Expand All @@ -133,7 +133,7 @@ def __init__(self, filetowatch, callback):
def watchfile(self):
return self._filetowatch

def on_any_event(self, event):
def on_closed(self, event):
_log.debug("Calling callback on event {}. Calling {}".format(event, self._callback))
try:
self._callback(self._filetowatch)
Expand Down
14 changes: 0 additions & 14 deletions volttrontesting/platform/auth_tests/test_auth_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,20 +375,6 @@ def test_auth_rpc_method_remove(auth_instance):
assert entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]}


@pytest.mark.control
def test_group_cmds(auth_instance):
"""Test add-group, list-groups, update-group, and remove-group"""
_run_group_or_role_cmds(auth_instance, _add_group, _list_groups,
_update_group, _remove_group)


@pytest.mark.control
def test_role_cmds(auth_instance):
"""Test add-role, list-roles, update-role, and remove-role"""
_run_group_or_role_cmds(auth_instance, _add_role, _list_roles,
_update_role, _remove_role)


def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn):
expected = []
key = '0'
Expand Down
174 changes: 174 additions & 0 deletions volttrontesting/platform/auth_tests/test_auth_group_roles.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@

import os
import re
import subprocess

import gevent
import pytest
from mock import MagicMock
from volttron.platform.auth.auth_protocols.auth_zmq import ZMQAuthorization, ZMQServerAuthentication

from volttrontesting.platform.auth_tests.conftest import assert_auth_entries_same
from volttrontesting.utils.platformwrapper import with_os_environ
from volttrontesting.utils.utils import AgentMock
from volttron.platform.vip.agent import Agent
from volttron.platform.auth import AuthService
from volttron.platform.auth import AuthEntry
from volttron.platform import jsonapi

@pytest.fixture(autouse=True)
def auth_instance(volttron_instance):
if not volttron_instance.auth_enabled:
pytest.skip("AUTH tests are not applicable if auth is disabled")
with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'r') as f:
auth_file = jsonapi.load(f)
print(auth_file)
try:
yield volttron_instance
finally:
with with_os_environ(volttron_instance.env):
with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'w') as f:
jsonapi.dump(auth_file, f)


def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn):
expected = []
key = '0'
values = ['0', '1']
expected.extend(values)

add_fn(platform, key, values)
gevent.sleep(4)
keys = list_fn(platform)
assert set(keys[key]) == set(expected)

# Update add single value
values = ['2']
expected.extend(values)
update_fn(platform, key, values)
gevent.sleep(2)
keys = list_fn(platform)
assert set(keys[key]) == set(expected)

# Update add multiple values
values = ['3', '4']
expected.extend(values)
update_fn(platform, key, values)
gevent.sleep(2)
keys = list_fn(platform)
assert set(keys[key]) == set(expected)

# Update remove single value
value = '0'
expected.remove(value)
update_fn(platform, key, [value], remove=True)
gevent.sleep(2)
keys = list_fn(platform)
assert set(keys[key]) == set(expected)

# Update remove single value
values = ['1', '2']
for value in values:
expected.remove(value)
update_fn(platform, key, values, remove=True)
gevent.sleep(2)
keys = list_fn(platform)
assert set(keys[key]) == set(expected)

# Remove key
remove_fn(platform, key)
gevent.sleep(2)
keys = list_fn(platform)
assert key not in keys



def _add_group_or_role(platform, cmd, name, list_):
with with_os_environ(platform.env):
args = ['volttron-ctl', 'auth', cmd, name]
args.extend(list_)
p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
p.communicate()
assert p.returncode == 0


def _add_group(platform, group, roles):
_add_group_or_role(platform, 'add-group', group, roles)


def _add_role(platform, role, capabilities):
_add_group_or_role(platform, 'add-role', role, capabilities)


def _list_groups_or_roles(platform, cmd):
with with_os_environ(platform.env):
output = subprocess.check_output(['volttron-ctl', 'auth', cmd],
env=platform.env, universal_newlines=True)
# For these tests don't use names that contain space, [, comma, or '
output = output.replace('[', '').replace("'", '').replace(']', '')
output = output.replace(',', '')
lines = output.split('\n')

dict_ = {}
for line in lines[2:-1]: # skip two header lines and last (empty) line
list_ = ' '.join(line.split()).split() # combine multiple spaces
dict_[list_[0]] = list_[1:]
return dict_


def _list_groups(platform):
return _list_groups_or_roles(platform, 'list-groups')


def _list_roles(platform):
return _list_groups_or_roles(platform, 'list-roles')


def _update_group_or_role(platform, cmd, key, values, remove):
with with_os_environ(platform.env):
args = ['volttron-ctl', 'auth', cmd, key]
args.extend(values)
if remove:
args.append('--remove')
p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
p.communicate()
assert p.returncode == 0


def _update_group(platform, group, roles, remove=False):
_update_group_or_role(platform, 'update-group', group, roles, remove)


def _update_role(platform, role, caps, remove=False):
_update_group_or_role(platform, 'update-role', role, caps, remove)


def _remove_group_or_role(platform, cmd, key):
with with_os_environ(platform.env):
args = ['volttron-ctl', 'auth', cmd, key]
p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True)
p.communicate()
assert p.returncode == 0


def _remove_group(platform, group):
_remove_group_or_role(platform, 'remove-group', group)


def _remove_role(platform, role):
_remove_group_or_role(platform, 'remove-role', role)


@pytest.mark.control
def test_group_cmds(auth_instance):
"""Test add-group, list-groups, update-group, and remove-group"""
_run_group_or_role_cmds(auth_instance, _add_group, _list_groups,
_update_group, _remove_group)


@pytest.mark.control
def test_role_cmds(auth_instance):
"""Test add-role, list-roles, update-role, and remove-role"""
_run_group_or_role_cmds(auth_instance, _add_role, _list_roles,
_update_role, _remove_role)

Loading

0 comments on commit 7b5339b

Please sign in to comment.