Skip to content

Commit

Permalink
Added dummy identity provider to remove Keystone dependency
Browse files Browse the repository at this point in the history
during testing

The identity provider used by `esi_leap` is now abstracted with
a `BaseIDP` ABC, and implemented by two subclasses: A `KeystoneIDP`
class which mostly copied the code from `common/keystone.py`,
and `DummyIDP` which mocks a real IDP

The IDP used by `esi_leap` can be set by overriding the `idp_type` CONF value like so:

CONF.set_override(
       "idp_type", "dummy_idp", group="esi"
 )

As a consequence of abstracting the IDP, `common/keystone.py`
has been removed. All references to `common/keystone.py` and
the functions defined in it has been appropriately changed.

New unit tests have been added for the keystone and dummy IDP clients
  • Loading branch information
QuanMPhm committed Nov 13, 2024
1 parent 0f7ed83 commit 2b31ea8
Show file tree
Hide file tree
Showing 23 changed files with 346 additions and 173 deletions.
6 changes: 2 additions & 4 deletions esi_leap/api/controllers/v1/event.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from esi_leap.api.controllers import types
from esi_leap.api.controllers.v1 import utils
from esi_leap.common import exception
from esi_leap.common import keystone
from esi_leap.common import idp
import esi_leap.conf
from esi_leap.objects import event as event_obj
from esi_leap.resource_objects import get_resource_object
Expand Down Expand Up @@ -82,9 +82,7 @@ def get_all(
lessee_or_owner_id = cdict["project_id"]

if lessee_or_owner_id is not None:
lessee_or_owner_id = keystone.get_project_uuid_from_ident(
lessee_or_owner_id
)
lessee_or_owner_id = idp.get_project_uuid_from_ident(lessee_or_owner_id)

if resource_uuid is not None:
if resource_type is None:
Expand Down
10 changes: 5 additions & 5 deletions esi_leap/api/controllers/v1/lease.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from esi_leap.common import constants
from esi_leap.common import exception
from esi_leap.common import ironic
from esi_leap.common import keystone
from esi_leap.common import idp
from esi_leap.common import statuses
import esi_leap.conf
from esi_leap.objects import lease as lease_obj
Expand Down Expand Up @@ -120,10 +120,10 @@ def get_all(
cdict = request.to_policy_values()

if project_id is not None:
project_id = keystone.get_project_uuid_from_ident(project_id)
project_id = idp.get_project_uuid_from_ident(project_id)

if owner_id is not None:
owner_id = keystone.get_project_uuid_from_ident(owner_id)
owner_id = idp.get_project_uuid_from_ident(owner_id)

if resource_uuid is not None:
if resource_type is None:
Expand Down Expand Up @@ -156,7 +156,7 @@ def get_all(

with concurrent.futures.ThreadPoolExecutor() as executor:
f1 = executor.submit(ironic.get_node_list)
f2 = executor.submit(keystone.get_project_list)
f2 = executor.submit(idp.get_project_list)
node_list = f1.result()
project_list = f2.result()

Expand Down Expand Up @@ -196,7 +196,7 @@ def post(self, new_lease):
lease_dict["resource_uuid"] = resource.get_uuid()

if "project_id" in lease_dict:
lease_dict["project_id"] = keystone.get_project_uuid_from_ident(
lease_dict["project_id"] = idp.get_project_uuid_from_ident(
lease_dict["project_id"]
)

Expand Down
12 changes: 6 additions & 6 deletions esi_leap/api/controllers/v1/node.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from esi_leap.api.controllers import base
from esi_leap.api.controllers import types
from esi_leap.common import ironic
from esi_leap.common import keystone
from esi_leap.common import idp
from esi_leap.common import statuses
import esi_leap.conf
from esi_leap.objects import lease as lease_obj
Expand Down Expand Up @@ -83,9 +83,9 @@ def get_all(self, resource_class=None, owner=None, lessee=None):
context = pecan.request.context

if owner is not None:
owner = keystone.get_project_uuid_from_ident(owner)
owner = idp.get_project_uuid_from_ident(owner)
if lessee is not None:
lessee = keystone.get_project_uuid_from_ident(lessee)
lessee = idp.get_project_uuid_from_ident(lessee)

filter_args = {
"resource_class": resource_class,
Expand All @@ -99,7 +99,7 @@ def get_all(self, resource_class=None, owner=None, lessee=None):
with concurrent.futures.ThreadPoolExecutor() as executor:
filter_args = {k: v for k, v in filter_args.items() if v is not None}
f1 = executor.submit(ironic.get_node_list, context, **filter_args)
f2 = executor.submit(keystone.get_project_list)
f2 = executor.submit(idp.get_project_list)
nodes = f1.result()
project_list = f2.result()

Expand Down Expand Up @@ -139,8 +139,8 @@ def get_all(self, resource_class=None, owner=None, lessee=None):
resource_class=node.resource_class,
properties=ironic.get_condensed_properties(node.properties),
maintenance=str(node.maintenance),
owner=keystone.get_project_name(node.owner, project_list),
lessee=keystone.get_project_name(node.lessee, project_list),
owner=idp.get_project_name(node.owner, project_list),
lessee=idp.get_project_name(node.lessee, project_list),
future_offers=f_offer_uuids,
future_leases=f_lease_uuids,
)
Expand Down
8 changes: 4 additions & 4 deletions esi_leap/api/controllers/v1/offer.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from esi_leap.api.controllers.v1 import utils
from esi_leap.common import exception
from esi_leap.common import ironic
from esi_leap.common import keystone
from esi_leap.common import idp
from esi_leap.common import statuses
import esi_leap.conf
from esi_leap.objects import lease as lease_obj
Expand Down Expand Up @@ -124,7 +124,7 @@ def get_all(
utils.policy_authorize("esi_leap:offer:get_all", cdict, cdict)

if project_id is not None:
project_id = keystone.get_project_uuid_from_ident(project_id)
project_id = idp.get_project_uuid_from_ident(project_id)

if resource_uuid is not None:
if resource_type is None:
Expand Down Expand Up @@ -193,7 +193,7 @@ def get_all(
node_list = None
with concurrent.futures.ThreadPoolExecutor() as executor:
f1 = executor.submit(ironic.get_node_list)
f2 = executor.submit(keystone.get_project_list)
f2 = executor.submit(idp.get_project_list)
node_list = f1.result()
project_list = f2.result()

Expand Down Expand Up @@ -231,7 +231,7 @@ def post(self, new_offer):
offer_dict["resource_uuid"] = resource.get_uuid()

if "lessee_id" in offer_dict:
offer_dict["lessee_id"] = keystone.get_project_uuid_from_ident(
offer_dict["lessee_id"] = idp.get_project_uuid_from_ident(
offer_dict["lessee_id"]
)

Expand Down
12 changes: 6 additions & 6 deletions esi_leap/api/controllers/v1/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import datetime

from esi_leap.common import exception
from esi_leap.common import keystone
from esi_leap.common import idp
from esi_leap.common import policy
from esi_leap.objects import lease as lease_obj
from esi_leap.objects import offer as offer_obj
Expand Down Expand Up @@ -148,7 +148,7 @@ def check_offer_lessee(cdict, offer):
if offer.lessee_id is None or offer.project_id == project_id:
return

if offer.lessee_id not in keystone.get_parent_project_id_tree(project_id):
if offer.lessee_id not in idp.get_parent_project_id_tree(project_id):
resource_policy_authorize(
"esi_leap:offer:offer_admin", cdict, cdict, "offer", offer.uuid
)
Expand All @@ -159,8 +159,8 @@ def offer_get_dict_with_added_info(offer, project_list=None, node_list=None):

o = offer.to_dict()
o["availabilities"] = offer.get_availabilities()
o["project"] = keystone.get_project_name(offer.project_id, project_list)
o["lessee"] = keystone.get_project_name(offer.lessee_id, project_list)
o["project"] = idp.get_project_name(offer.project_id, project_list)
o["lessee"] = idp.get_project_name(offer.lessee_id, project_list)
o["resource"] = resource.get_name(node_list)
o["resource_class"] = resource.get_resource_class(node_list)
o["resource_properties"] = resource.get_properties(node_list)
Expand All @@ -171,8 +171,8 @@ def lease_get_dict_with_added_info(lease, project_list=None, node_list=None):
resource = lease.resource_object()

lease_dict = lease.to_dict()
lease_dict["project"] = keystone.get_project_name(lease.project_id, project_list)
lease_dict["owner"] = keystone.get_project_name(lease.owner_id, project_list)
lease_dict["project"] = idp.get_project_name(lease.project_id, project_list)
lease_dict["owner"] = idp.get_project_name(lease.owner_id, project_list)
lease_dict["resource"] = resource.get_name(node_list)
lease_dict["resource_class"] = resource.get_resource_class(node_list)
lease_dict["resource_properties"] = resource.get_properties(node_list)
Expand Down
36 changes: 19 additions & 17 deletions esi_leap/common/keystone.py → esi_leap/common/idp.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,38 +10,40 @@
# License for the specific language governing permissions and limitations
# under the License.

from keystoneauth1 import loading as ks_loading
from keystoneclient import client as keystone_client
from oslo_utils import uuidutils

from esi_leap.common import exception
import esi_leap.conf
from esi_leap.common.idp_clients import baseIDP
from esi_leap.common.idp_clients import keystoneIDP # noqa: F401
from esi_leap.common.idp_clients import dummyIDP # noqa: F401


CONF = esi_leap.conf.CONF
_cached_keystone_client = None
IDP_TYPE_MAP = {idp.idp_type: idp for idp in baseIDP.BaseIDP.__subclasses__()}
_cached_idp_client = None
_cached_project_list = None


def get_keystone_client():
global _cached_keystone_client
if _cached_keystone_client is not None:
return _cached_keystone_client
def get_idp_client():
global _cached_idp_client
if _cached_idp_client is not None:
return _cached_idp_client

auth_plugin = ks_loading.load_auth_from_conf_options(CONF, "keystone")
sess = ks_loading.load_session_from_conf_options(CONF, "keystone", auth=auth_plugin)
cli = keystone_client.Client(session=sess)
_cached_keystone_client = cli
# Get Client config option
idp_type = CONF.esi.idp_type
cli = IDP_TYPE_MAP[idp_type]()
_cached_idp_client = cli

return cli


def get_parent_project_id_tree(project_id):
ks_client = get_keystone_client()
project = ks_client.projects.get(project_id)
idp = get_idp_client()
project = idp.get_projects(project_id)
project_ids = [project.id]
while project.parent_id is not None:
project = ks_client.projects.get(project.parent_id)
project = idp.get_projects(project.parent_id)
project_ids.append(project.id)
return project_ids

Expand All @@ -50,21 +52,21 @@ def get_project_uuid_from_ident(project_ident):
if uuidutils.is_uuid_like(project_ident):
return project_ident
else:
projects = get_keystone_client().projects.list(name=project_ident)
projects = get_idp_client().list_projects(name=project_ident)
if len(projects) > 0:
# projects have unique names
return projects[0].id
raise exception.ProjectNoSuchName(name=project_ident)


def get_project_list():
return get_keystone_client().projects.list()
return get_idp_client().list_projects()


def get_project_name(project_id, project_list=None):
if project_id:
if project_list is None:
project = get_keystone_client().projects.get(project_id)
project = get_idp_client().get_projects(project_id)
else:
project = next(
(p for p in project_list if getattr(p, "id") == project_id), None
Expand Down
Empty file.
23 changes: 23 additions & 0 deletions esi_leap/common/idp_clients/baseIDP.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import abc


class BaseIDP(abc.ABC):
@abc.abstractmethod
def get_projects(self, project_id):
pass

@abc.abstractmethod
def list_projects(self, **kwargs):
pass
40 changes: 40 additions & 0 deletions esi_leap/common/idp_clients/dummyIDP.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from keystoneclient.v3.projects import Project
from keystoneclient.v3.projects import ProjectManager
from keystoneclient.v3.users import User
from keystoneclient.v3.users import UserManager

from esi_leap.common.idp_clients import baseIDP


class DummyIDP(baseIDP.BaseIDP):
idp_type = "dummy_idp"
dummy_project_dict = {}
dummy_user_dict = {}

def get_projects(self, project_id):
return self.dummy_project_dict[project_id]

def list_projects(self, **kwargs):
return [project for project in self.dummy_project_dict.values()]

def add_project(self, id, name, parent_id):
self.dummy_project_dict[id] = Project(
ProjectManager, {"id": id, "name": name, "parent_id": parent_id}
)

def add_user(self, id, name, project_id):
self.dummy_user_dict[id] = User(
UserManager, {"id": id, "name": name, "project_id": project_id}
)
36 changes: 36 additions & 0 deletions esi_leap/common/idp_clients/keystoneIDP.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from keystoneauth1 import loading as ks_loading
from keystoneclient import client as keystone_client

from esi_leap.common.idp_clients import baseIDP
import esi_leap.conf

CONF = esi_leap.conf.CONF


class KeystoneIDP(baseIDP.BaseIDP):
idp_type = "keystone_idp"

def __init__(self) -> None:
auth_plugin = ks_loading.load_auth_from_conf_options(CONF, "keystone")
sess = ks_loading.load_session_from_conf_options(
CONF, "keystone", auth=auth_plugin
)
self.cli = keystone_client.Client(session=sess)

def get_projects(self, project_id):
return self.cli.projects.get(project_id)

def list_projects(self, **kwargs):
return self.cli.projects.list(**kwargs)
2 changes: 2 additions & 0 deletions esi_leap/conf/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

from esi_leap.conf import api
from esi_leap.conf import dummy_node
from esi_leap.conf import esi
from esi_leap.conf import ironic
from esi_leap.conf import keystone
from esi_leap.conf import netconf
Expand All @@ -33,3 +34,4 @@
notification.register_opts(CONF)
pecan.register_opts(CONF)
serialconsoleproxy.register_opts(CONF)
esi.register_opts(CONF)
23 changes: 23 additions & 0 deletions esi_leap/conf/esi.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from oslo_config import cfg

opts = [
cfg.StrOpt("idp_type", default="keystone_idp"),
]

api_group = cfg.OptGroup("esi", title="ESI Options")


def register_opts(conf):
conf.register_opts(opts, group=api_group)
Loading

0 comments on commit 2b31ea8

Please sign in to comment.