Skip to content

Commit

Permalink
refactor(apply): removes resolver cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@pallabpain
pallabpain committed Jul 31, 2024
1 parent 6f69cac commit 2c85639
Show file tree
Hide file tree
Showing 22 changed files with 520 additions and 1,281 deletions.
2 changes: 1 addition & 1 deletion riocli/apply/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ def delete(
raise SystemExit(1)

rc = Applier(glob_files, abs_values, abs_secrets)
rc.parse_dependencies(check_missing=False, delete=True)
rc.parse_dependencies()

if not silent and not dryrun:
click.confirm("Do you want to proceed?", default=True, abort=True)
Expand Down
182 changes: 58 additions & 124 deletions riocli/apply/parse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@
import click
import jinja2
import yaml
from munch import munchify

from riocli.apply.resolver import ResolverCache
from riocli.config import Configuration
from riocli.apply.util import get_model
from riocli.apply.util import message_with_prompt
from riocli.constants import Colors, Symbols
from riocli.utils import dump_all_yaml, run_bash
from riocli.utils.graph import Graphviz
Expand All @@ -35,14 +36,10 @@ class Applier(object):
def __init__(self, files: typing.List, values, secrets):
self.environment = None
self.input_file_paths = files
self.config = Configuration()
self.client = self.config.new_client()
self.dependencies = {}
self.objects = {}
self.resolved_objects = {}
self.files = {}
self.graph = TopologicalSorter()
self.rc = ResolverCache(self.client)
self.secrets = {}
self.values = {}
self.diagram = Graphviz(direction='LR', format='svg')
Expand All @@ -59,10 +56,6 @@ def __init__(self, files: typing.List, values, secrets):

self._process_file_list(files)

# Public Functions
def order(self):
return self.graph.static_order()

@with_spinner(text='Applying...', timer=True)
def apply(self, *args, **kwargs):
spinner = kwargs.get('spinner')
Expand Down Expand Up @@ -94,9 +87,6 @@ def worker():
try:
self._apply_manifest(o, *args, **kwargs)
except Exception as ex:
click.secho(
'[Err] Object "{}" apply failed. Apply will not progress further.'.format(
o, str(ex)))
done_queue.put(ex)
continue

Expand Down Expand Up @@ -150,49 +140,66 @@ def print_resolved_manifests(self):
manifests = [o for _, o in self.objects.items()]
dump_all_yaml(manifests)

def parse_dependencies(
self,
check_missing=True,
delete=False,
template=False
):
number_of_objects = 0
for f, data in self.files.items():
def parse_dependencies(self):
for _, data in self.files.items():
for model in data:
key = self._get_object_key(model)
self._parse_dependency(key, model)
self._add_graph_node(key)
number_of_objects = number_of_objects + 1

if check_missing:
missing_resources = []
for key, item in self.resolved_objects.items():
if 'src' in item and item['src'] == 'missing':
missing_resources.append(key)

if missing_resources:
click.secho(
"Missing resources found in yaml. Please ensure the "
"following are either available in your YAML or created"
" on the server. {}".format(
set(missing_resources)), fg=Colors.RED)
def show_dependency_graph(self):
"""Lauches mermaid.live dependency graph"""
self.diagram.visualize()

raise SystemExit(1)
def _apply_manifest(self, obj_key: str, *args, **kwargs) -> None:
"""Instantiate and apply the object manifest"""
spinner = kwargs.get('spinner')
dryrun = kwargs.get("dryrun", False)

# Manifest Operations via base.py
def _apply_manifest(self, obj_key, *args, **kwargs):
obj = self.objects[obj_key]
cls = ResolverCache.get_model(obj)
ist = cls.from_dict(self.client, obj)
setattr(ist, 'rc', ResolverCache(self.client))
ist.apply(self.client, *args, **kwargs)
kls = get_model(obj)
kls.validate(obj)
ist = kls(munchify(obj))

message_with_prompt("{} Applying {}...".format(
Symbols.WAITING, obj_key), fg=Colors.YELLOW, spinner=spinner)

try:
if not dryrun:
ist.apply(*args, **kwargs)
except Exception as ex:
message_with_prompt("{} Failed to apply {}. Error: {}".format(
Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner)
raise ex

message_with_prompt("{} Applied {}.".format(
Symbols.SUCCESS, obj_key), fg=Colors.YELLOW, spinner=spinner)

def _delete_manifest(self, obj_key: str, *args, **kwargs) -> None:
"""Instantiate and delete the object manifest"""
spinner = kwargs.get('spinner')
dryrun = kwargs.get("dryrun", False)

def _delete_manifest(self, obj_key, *args, **kwargs):
obj = self.objects[obj_key]
cls = ResolverCache.get_model(obj)
ist = cls.from_dict(self.client, obj)
setattr(ist, 'rc', ResolverCache(self.client))
ist.delete(self.client, obj, *args, **kwargs)
kls = get_model(obj)
kls.validate(obj)
ist = kls(munchify(obj))

message_with_prompt("{} Deleting {}...".format(
Symbols.WAITING, obj_key), fg=Colors.YELLOW, spinner=spinner)

# TODO(pallab): Handle retain policy

try:
if not dryrun:
ist.delete(*args, **kwargs)
except Exception as ex:
message_with_prompt("{} Failed to delete {}. Error: {}".format(
Symbols.ERROR, obj_key, str(ex)), fg=Colors.RED, spinner=spinner)
raise ex

message_with_prompt("{} Deleted {}.".format(
Symbols.SUCCESS, obj_key), fg=Colors.YELLOW, spinner=spinner)

# File Loading Operations

Expand Down Expand Up @@ -281,6 +288,11 @@ def _add_graph_edge(self, dependent_key, key):

# Dependency Resolution
def _parse_dependency(self, dependent_key, model):
# TODO(pallab): let resources determine their own dependencies and return them
# kls = get_model(model)
# for dependency in kls.parse_dependencies(model):
# self._resolve_dependency(dependent_key, dependency)

for key, value in model.items():
if key == "depends":
if 'kind' in value and value.get('kind'):
Expand All @@ -306,86 +318,8 @@ def _resolve_dependency(self, dependent_key, dependency):
name_or_guid = dependency.get('nameOrGUID')
key = '{}:{}'.format(kind, name_or_guid)

self._initialize_kind_dependency(kind)
guid = ResolverCache._maybe_guid(kind, name_or_guid)

obj_list = self.rc.list_objects(kind)
for obj in obj_list:
obj_guid = self._get_attr(obj, ResolverCache.GUID_KEYS)
obj_name = self._get_attr(obj, ResolverCache.NAME_KEYS)

if kind == 'package':
if guid and obj_guid == guid:
self._add_remote_object_to_resolve_tree(
dependent_key, obj_guid, dependency, obj)

if (name_or_guid == obj_name) and ('version' in dependency and
obj.metadata.version == dependency.get('version')):
self._add_remote_object_to_resolve_tree(
dependent_key, obj_guid, dependency, obj)

# Special handling for Static route since it doesn't have a name field.
# StaticRoute sends a URLPrefix field with name being the prefix along with short org guid.
elif kind == 'staticroute' and name_or_guid in obj_name:
self._add_remote_object_to_resolve_tree(
dependent_key, obj_guid, dependency, obj)

elif (guid and obj_guid == guid) or (name_or_guid == obj_name):
self._add_remote_object_to_resolve_tree(
dependent_key, obj_guid, dependency, obj)

self.dependencies[kind][name_or_guid] = {'local': True}
self._add_graph_edge(dependent_key, key)

if key not in self.resolved_objects:
self.resolved_objects[key] = {'src': 'missing'}

def _add_remote_object_to_resolve_tree(self, dependent_key, guid,
dependency, obj):
kind = dependency.get('kind')
name_or_guid = dependency.get('nameOrGUID')
key = '{}:{}'.format(kind, name_or_guid)
self.dependencies[kind][name_or_guid] = {
'guid': guid, 'raw': obj, 'local': False}
if key not in self.resolved_objects:
self.resolved_objects[key] = {}
self.resolved_objects[key]['guid'] = guid
self.resolved_objects[key]['raw'] = obj
self.resolved_objects[key]['src'] = 'remote'

self._add_graph_edge(dependent_key, key)

dependency['guid'] = guid
if kind.lower() == "disk":
dependency['depGuid'] = obj.metadata.guid

if kind.lower() == "deployment":
dependency['guid'] = obj.metadata.guid

def _initialize_kind_dependency(self, kind):
if not self.dependencies.get(kind):
self.dependencies[kind] = {}

def show_dependency_graph(self):
"""Lauches mermaid.live dependency graph"""
self.diagram.visualize()

# Utils
@staticmethod
def _get_attr(obj, accept_keys):
metadata = None

if hasattr(obj, 'metadata'):
metadata = getattr(obj, 'metadata')

for key in accept_keys:
if hasattr(obj, key):
return getattr(obj, key)
if metadata is not None and hasattr(metadata, key):
return getattr(metadata, key)

raise Exception('guid resolve failed')

@staticmethod
def _get_object_key(obj: dict) -> str:
kind = obj.get('kind').lower()
Expand Down
Loading

0 comments on commit 2c85639

Please sign in to comment.