From 19db9d48d2f4d56753f44a0c9a9a8307e3107e24 Mon Sep 17 00:00:00 2001 From: Jan Milenkov Date: Thu, 20 Feb 2025 19:40:08 +0200 Subject: [PATCH] feat(multi-cluster): Update 'relay destroy' and 'network destroy' to support multiple clusters (#1424) Signed-off-by: instamenta Signed-off-by: Zhan Milenkov --- src/commands/base.ts | 2 + src/commands/mirror_node.ts | 4 +- src/commands/network.ts | 107 +++++++++++------- src/commands/relay.ts | 27 +++-- src/core/account_manager.ts | 1 - .../config/remote/remote_config_manager.ts | 101 +++++++++++++++-- 6 files changed, 180 insertions(+), 62 deletions(-) diff --git a/src/commands/base.ts b/src/commands/base.ts index 95daaae5a..30de03811 100644 --- a/src/commands/base.ts +++ b/src/commands/base.ts @@ -407,6 +407,7 @@ export abstract class BaseCommand extends ShellRunner { /** * Gets a list of distinct contexts from the consensus nodes * @returns an array of context strings + * @deprecated use one inside remote config */ public getContexts(): string[] { const contexts: string[] = []; @@ -421,6 +422,7 @@ export abstract class BaseCommand extends ShellRunner { /** * Gets a list of distinct cluster references from the consensus nodes * @returns an object of cluster references + * @deprecated use one inside remote config */ public getClusterRefs(): ClusterRefs { const clustersRefs: ClusterRefs = {}; diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index b35f8a01e..0b6126994 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -28,9 +28,8 @@ import chalk from 'chalk'; import {type CommandFlag} from '../types/flag_types.js'; import {PvcRef} from '../core/kube/resources/pvc/pvc_ref.js'; import {PvcName} from '../core/kube/resources/pvc/pvc_name.js'; -import {type DeploymentName} from '../core/config/remote/types.js'; +import {type ClusterRef, type DeploymentName} from '../core/config/remote/types.js'; import {extractContextFromConsensusNodes} from '../core/helpers.js'; -import {node} from 'globals'; interface MirrorNodeDeployConfigClass { chartDirectory: string; @@ -623,6 +622,7 @@ export class MirrorNodeCommand extends BaseCommand { namespace: NamespaceName; clusterContext: string; isChartInstalled: boolean; + clusterRef?: Optional; }; } diff --git a/src/commands/network.ts b/src/commands/network.ts index 7ee7aa8ef..81b90c66d 100644 --- a/src/commands/network.ts +++ b/src/commands/network.ts @@ -25,7 +25,7 @@ import {ConsensusNodeStates} from '../core/config/remote/enumerations.js'; import {EnvoyProxyComponent} from '../core/config/remote/components/envoy_proxy_component.js'; import {HaProxyComponent} from '../core/config/remote/components/ha_proxy_component.js'; import {v4 as uuidv4} from 'uuid'; -import {type SoloListrTask} from '../types/index.js'; +import {type SoloListrTask, type SoloListrTaskWrapper} from '../types/index.js'; import {NamespaceName} from '../core/kube/resources/namespace/namespace_name.js'; import {PvcRef} from '../core/kube/resources/pvc/pvc_ref.js'; import {PvcName} from '../core/kube/resources/pvc/pvc_name.js'; @@ -87,6 +87,18 @@ export interface NetworkDeployConfigClass { clusterRefs: ClusterRefs; } +interface NetworkDestroyContext { + config: { + deletePvcs: boolean; + deleteSecrets: boolean; + namespace: NamespaceName; + enableTimeout: boolean; + force: boolean; + contexts: string[]; + }; + checkTimeout: boolean; +} + export class NetworkCommand extends BaseCommand { private readonly keyManager: KeyManager; private readonly platformInstaller: PlatformInstaller; @@ -709,37 +721,58 @@ export class NetworkCommand extends BaseCommand { return config; } - async destroyTask(ctx: any, task: any) { + async destroyTask(ctx: NetworkDestroyContext, task: SoloListrTaskWrapper) { const self = this; task.title = `Uninstalling chart ${constants.SOLO_DEPLOYMENT_CHART}`; - await self.chartManager.uninstall( - ctx.config.namespace, - constants.SOLO_DEPLOYMENT_CHART, - this.k8Factory.default().contexts().readCurrent(), + + // Uninstall all 'solo deployment' charts for each cluster using the contexts + await Promise.all( + ctx.config.contexts.map(context => { + return self.chartManager.uninstall( + ctx.config.namespace, + constants.SOLO_DEPLOYMENT_CHART, + this.k8Factory.getK8(context).contexts().readCurrent(), + ); + }), ); + // Delete PVCs inside each cluster if (ctx.config.deletePvcs) { - const pvcs = await self.k8Factory.default().pvcs().list(ctx.config.namespace, []); task.title = `Deleting PVCs in namespace ${ctx.config.namespace}`; - if (pvcs) { - for (const pvc of pvcs) { - await self.k8Factory - .default() - .pvcs() - .delete(PvcRef.of(ctx.config.namespace, PvcName.of(pvc))); - } - } + + await Promise.all( + ctx.config.contexts.map(async context => { + // Fetch all PVCs inside the namespace using the context + const pvcs = await this.k8Factory.getK8(context).pvcs().list(ctx.config.namespace, []); + + // Delete all if found + return Promise.all( + pvcs.map(pvc => + this.k8Factory + .getK8(context) + .pvcs() + .delete(PvcRef.of(ctx.config.namespace, PvcName.of(pvc))), + ), + ); + }), + ); } + // Delete Secrets inside each cluster if (ctx.config.deleteSecrets) { task.title = `Deleting secrets in namespace ${ctx.config.namespace}`; - const secrets = await self.k8Factory.default().secrets().list(ctx.config.namespace); - if (secrets) { - for (const secret of secrets) { - await self.k8Factory.default().secrets().delete(ctx.config.namespace, secret.name); - } - } + await Promise.all( + ctx.config.contexts.map(async context => { + // Fetch all Secrets inside the namespace using the context + const secrets = await this.k8Factory.getK8(context).secrets().list(ctx.config.namespace); + + // Delete all if found + return Promise.all( + secrets.map(secret => this.k8Factory.getK8(context).secrets().delete(ctx.config.namespace, secret.name)), + ); + }), + ); } } @@ -1077,19 +1110,8 @@ export class NetworkCommand extends BaseCommand { const self = this; const lease = await self.leaseManager.create(); - interface Context { - config: { - deletePvcs: boolean; - deleteSecrets: boolean; - namespace: NamespaceName; - enableTimeout: boolean; - force: boolean; - }; - checkTimeout: boolean; - } - let networkDestroySuccess = true; - const tasks = new Listr( + const tasks = new Listr( [ { title: 'Initialize', @@ -1108,14 +1130,14 @@ export class NetworkCommand extends BaseCommand { self.configManager.update(argv); await self.configManager.executePrompt(task, [flags.deletePvcs, flags.deleteSecrets]); - const namespace = await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task); ctx.config = { deletePvcs: self.configManager.getFlag(flags.deletePvcs) as boolean, deleteSecrets: self.configManager.getFlag(flags.deleteSecrets) as boolean, - namespace, + namespace: await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task), enableTimeout: self.configManager.getFlag(flags.enableTimeout) as boolean, force: self.configManager.getFlag(flags.force) as boolean, + contexts: this.getContexts(), }; return ListrLease.newAcquireLeaseTask(lease, task); @@ -1125,18 +1147,22 @@ export class NetworkCommand extends BaseCommand { title: 'Running sub-tasks to destroy network', task: async (ctx, task) => { if (ctx.config.enableTimeout) { - const timeoutId = setTimeout(() => { + const timeoutId = setTimeout(async () => { const message = `\n\nUnable to finish network destroy in ${constants.NETWORK_DESTROY_WAIT_TIMEOUT} seconds\n\n`; self.logger.error(message); self.logger.showUser(chalk.red(message)); networkDestroySuccess = false; if (ctx.config.deletePvcs && ctx.config.deleteSecrets && ctx.config.force) { - self.k8Factory.default().namespaces().delete(ctx.config.namespace); + await Promise.all( + ctx.config.contexts.map(context => + self.k8Factory.getK8(context).namespaces().delete(ctx.config.namespace), + ), + ); } else { // If the namespace is not being deleted, // remove all components data from the remote configuration - self.remoteConfigManager.deleteComponents(); + await self.remoteConfigManager.deleteComponents(); } }, constants.NETWORK_DESTROY_WAIT_TIMEOUT * 1_000); @@ -1157,10 +1183,11 @@ export class NetworkCommand extends BaseCommand { try { await tasks.run(); - } catch (e: Error | unknown) { + } catch (e) { throw new SoloError('Error destroying network', e); } finally { - await lease.release(); + // If the namespace is deleted, the lease can't be released + await lease.release().catch(); } return networkDestroySuccess; diff --git a/src/commands/relay.ts b/src/commands/relay.ts index b6a8c7c08..ea4ed2609 100644 --- a/src/commands/relay.ts +++ b/src/commands/relay.ts @@ -57,7 +57,7 @@ export class RelayCommand extends BaseCommand { } static get DESTROY_FLAGS_LIST() { - return [flags.chartDirectory, flags.deployment, flags.nodeAliasesUnparsed]; + return [flags.chartDirectory, flags.deployment, flags.nodeAliasesUnparsed, flags.clusterRef]; } async prepareValuesArg( @@ -357,6 +357,8 @@ export class RelayCommand extends BaseCommand { nodeAliases: NodeAliases; releaseName: string; isChartInstalled: boolean; + clusterRef: Optional; + context: Optional; } interface Context { @@ -370,24 +372,32 @@ export class RelayCommand extends BaseCommand { task: async (ctx, task) => { // reset nodeAlias self.configManager.setFlag(flags.nodeAliasesUnparsed, ''); - self.configManager.update(argv); + + flags.disablePrompts([flags.clusterRef]); + await self.configManager.executePrompt(task, RelayCommand.DESTROY_FLAGS_LIST); - const namespace = await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task); // prompt if inputs are empty and set it in the context ctx.config = { chartDirectory: self.configManager.getFlag(flags.chartDirectory) as string, - namespace: namespace, + namespace: await resolveNamespaceFromDeployment(this.localConfig, this.configManager, task), nodeAliases: helpers.parseNodeAliases( self.configManager.getFlag(flags.nodeAliasesUnparsed) as string, ), + clusterRef: self.configManager.getFlag(flags.clusterRef) as string, } as RelayDestroyConfigClass; + if (ctx.config.clusterRef) { + const context = self.getClusterRefs()[ctx.config.clusterRef]; + if (context) ctx.config.context = context; + } + ctx.config.releaseName = this.prepareReleaseName(ctx.config.nodeAliases); ctx.config.isChartInstalled = await this.chartManager.isChartInstalled( ctx.config.namespace, ctx.config.releaseName, + ctx.config.context, ); self.logger.debug('Initialized config', {config: ctx.config}); @@ -403,10 +413,13 @@ export class RelayCommand extends BaseCommand { await this.chartManager.uninstall( config.namespace, config.releaseName, - this.k8Factory.default().contexts().readCurrent(), + this.k8Factory.getK8(ctx.config.context).contexts().readCurrent(), ); - this.logger.showList('Destroyed Relays', await self.chartManager.getInstalledCharts(config.namespace)); + this.logger.showList( + 'Destroyed Relays', + await self.chartManager.getInstalledCharts(config.namespace, config.context), + ); // reset nodeAliasesUnparsed self.configManager.setFlag(flags.nodeAliasesUnparsed, ''); @@ -423,7 +436,7 @@ export class RelayCommand extends BaseCommand { try { await tasks.run(); - } catch (e: Error | any) { + } catch (e) { throw new SoloError('Error uninstalling relays', e); } finally { await lease.release(); diff --git a/src/core/account_manager.ts b/src/core/account_manager.ts index 691615147..9700ca139 100644 --- a/src/core/account_manager.ts +++ b/src/core/account_manager.ts @@ -571,7 +571,6 @@ export class AccountManager { .pods() .list(namespace, ['solo.hedera.com/type=network-node']); for (const pod of pods) { - // eslint-disable-next-line no-prototype-builtins if (!pod.metadata?.labels?.hasOwnProperty('solo.hedera.com/node-name')) { // TODO Review why this fixes issue continue; diff --git a/src/core/config/remote/remote_config_manager.ts b/src/core/config/remote/remote_config_manager.ts index ef59cf5cb..b45324e39 100644 --- a/src/core/config/remote/remote_config_manager.ts +++ b/src/core/config/remote/remote_config_manager.ts @@ -11,7 +11,14 @@ import * as yaml from 'yaml'; import {ComponentsDataWrapper} from './components_data_wrapper.js'; import {RemoteConfigValidator} from './remote_config_validator.js'; import {type K8Factory} from '../../kube/k8_factory.js'; -import {type ClusterRef, type Context, type DeploymentName, type NamespaceNameAsString, type Version} from './types.js'; +import { + type ClusterRef, + type ClusterRefs, + type Context, + type DeploymentName, + type NamespaceNameAsString, + type Version, +} from './types.js'; import {type SoloLogger} from '../../logging.js'; import {type ConfigManager} from '../../config_manager.js'; import {type LocalConfig} from '../local_config.js'; @@ -22,12 +29,14 @@ import {inject, injectable} from 'tsyringe-neo'; import {patchInject} from '../../dependency_injection/container_helper.js'; import {ErrorMessages} from '../../error_messages.js'; import {CommonFlagsDataWrapper} from './common_flags_data_wrapper.js'; -import {type AnyObject} from '../../../types/aliases.js'; +import {type AnyObject, type NodeAlias} from '../../../types/aliases.js'; import {NamespaceName} from '../../kube/resources/namespace/namespace_name.js'; import {ResourceNotFoundError} from '../../kube/errors/resource_operation_errors.js'; import {InjectTokens} from '../../dependency_injection/inject_tokens.js'; import {Cluster} from './cluster.js'; import * as helpers from '../../helpers.js'; +import {ConsensusNode} from '../../model/consensus_node.js'; +import {Templates} from '../../templates.js'; import {promptTheUserForDeployment, resolveNamespaceFromDeployment} from '../../resolvers.js'; /** @@ -384,17 +393,20 @@ export class RemoteConfigManager { /** Replaces an existing ConfigMap in the Kubernetes cluster with the current remote configuration data. */ private async replaceConfigMap(): Promise { - await this.k8Factory - .default() - .configMaps() - .replace( - await this.getNamespace(), - constants.SOLO_REMOTE_CONFIGMAP_NAME, - constants.SOLO_REMOTE_CONFIGMAP_LABELS, - { + const contexts = this.getContexts(); + const namespace = await this.getNamespace(); + + await Promise.all( + contexts.map(context => { + const name = constants.SOLO_REMOTE_CONFIGMAP_NAME; + const labels = constants.SOLO_REMOTE_CONFIGMAP_LABELS; + const data = { 'remote-config-data': yaml.stringify(this.remoteConfig.toObject() as any), - }, - ); + }; + + return this.k8Factory.getK8(context).configMaps().replace(namespace, name, labels, data); + }), + ); } private async setDefaultNamespaceIfNotSet(argv: AnyObject): Promise { @@ -447,4 +459,69 @@ export class RemoteConfigManager { if (!ns) throw new MissingArgumentError('namespace is not set'); return ns; } + + //* Common Commands + + /** + * Get the consensus nodes from the remoteConfigManager and use the localConfig to get the context + * @returns an array of ConsensusNode objects + */ + public getConsensusNodes(): ConsensusNode[] { + const consensusNodes: ConsensusNode[] = []; + const clusters: Record = this.clusters; + + try { + if (!this?.components?.consensusNodes) return []; + } catch { + return []; + } + + // using the remoteConfigManager to get the consensus nodes + if (this?.components?.consensusNodes) { + Object.values(this.components.consensusNodes).forEach(node => { + consensusNodes.push( + new ConsensusNode( + node.name as NodeAlias, + node.nodeId, + node.namespace, + node.cluster, + // use local config to get the context + this.localConfig.clusterRefs[node.cluster], + clusters[node.cluster].dnsBaseDomain, + clusters[node.cluster].dnsConsensusNodePattern, + Templates.renderConsensusNodeFullyQualifiedDomainName( + node.name as NodeAlias, + node.nodeId, + node.namespace, + node.cluster, + clusters[node.cluster].dnsBaseDomain, + clusters[node.cluster].dnsConsensusNodePattern, + ), + ), + ); + }); + } + + // return the consensus nodes + return consensusNodes; + } + + /** + * Gets a list of distinct contexts from the consensus nodes. + * @returns an array of context strings. + */ + public getContexts(): string[] { + return [...new Set(this.getConsensusNodes().map(node => node.context))]; + } + + /** + * Gets a list of distinct cluster references from the consensus nodes. + * @returns an object of cluster references. + */ + public getClusterRefs(): ClusterRefs { + return this.getConsensusNodes().reduce((acc, node) => { + acc[node.cluster] ||= node.context; + return acc; + }, {} as ClusterRefs); + } }