From f301e190124c27a5762589ee0308c5f422ce6bc7 Mon Sep 17 00:00:00 2001 From: Pavel Silin Date: Tue, 27 Aug 2024 12:10:04 +0200 Subject: [PATCH] Issue #3602 fix for network tag removal and notification repeat (#3658) --- .../pipeline/common/MessageConstants.java | 1 + .../ResourceMonitoringManager.java | 40 ++++++++++--------- .../DockerContainerOperationManager.java | 5 ++- .../notification/NotificationManager.java | 17 -------- api/src/main/resources/messages.properties | 1 + 5 files changed, 27 insertions(+), 37 deletions(-) diff --git a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java index 2e8137c4ac..147a15f979 100644 --- a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java +++ b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java @@ -223,6 +223,7 @@ public final class MessageConstants { public static final String DEBUG_RUN_HAS_NOT_NODE_NAME = "debug.run.has.not.node.name"; public static final String DEBUG_MEMORY_METRICS = "debug.memory.metrics.received"; public static final String INFO_RUN_HIGH_NETWORK_CONSUMPTION_NOTIFY = "info.run.high.network.consumption.notify"; + public static final String INFO_RUN_HIGH_NETWORK_CONSUMPTION_ACTION = "info.run.high.network.consumption.action"; public static final String DEBUG_NETWORK_RUN_METRICS_RECEIVED = "debug.network.run.metrics.received"; public static final String DEBUG_RUN_NOT_NETWORK_CONSUMING = "debug.run.not.network.consuming"; public static final String DEBUG_RUN_NOT_NETWORK_CONSUMING_DISABLED = "debug.run.network.consuming.disabled"; diff --git a/api/src/main/java/com/epam/pipeline/manager/cluster/performancemonitoring/ResourceMonitoringManager.java b/api/src/main/java/com/epam/pipeline/manager/cluster/performancemonitoring/ResourceMonitoringManager.java index 434513f339..9017e5a2a2 100644 --- a/api/src/main/java/com/epam/pipeline/manager/cluster/performancemonitoring/ResourceMonitoringManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/cluster/performancemonitoring/ResourceMonitoringManager.java @@ -454,25 +454,24 @@ private void processHighNetworkConsumingRuns(final List runs) { private void processHighNetworkConsumingRun(PipelineRun run, int actionTimeout, NetworkConsumingRunAction action, - List> pipelinesToNotify, + List> runsToNotify, List runsToUpdateNotificationTime, Double bandwidth, List runsToUpdateTags) { - if (shouldPerformActionOnNetworkConsumingRun(run, actionTimeout)) { - performActionOnNetworkConsumingRun(run, action, bandwidth, pipelinesToNotify, - runsToUpdateNotificationTime); - return; - } if (Objects.isNull(run.getLastNetworkConsumptionNotificationTime())) { run.addTag(NETWORK_CONSUMING_LEVEL_HIGH, TRUE_VALUE_STRING); Optional.ofNullable(getTimestampTag(NETWORK_CONSUMING_LEVEL_HIGH)) .ifPresent(tag -> run.addTag(tag, DateUtils.nowUTCStr())); runsToUpdateTags.add(run); - run.setLastNetworkConsumptionNotificationTime(DateUtils.nowUTC()); - runsToUpdateNotificationTime.add(run); + + log.info(messageHelper.getMessage(MessageConstants.INFO_RUN_HIGH_NETWORK_CONSUMPTION_NOTIFY, + run.getPodId(), bandwidth)); + + performHighNetworkConsumingNotify(run, bandwidth, runsToNotify, runsToUpdateNotificationTime); + } else if (shouldPerformActionOnNetworkConsumingRun(run, actionTimeout)) { + performActionOnNetworkConsumingRun(run, action, bandwidth, runsToNotify, + runsToUpdateNotificationTime); } - pipelinesToNotify.add(new ImmutablePair<>(run, bandwidth)); - log.info(messageHelper.getMessage(MessageConstants.INFO_RUN_HIGH_NETWORK_CONSUMPTION_NOTIFY, - run.getPodId(), bandwidth)); + } private void processHighNetworkConsumingRuns(Map running, @@ -522,24 +521,27 @@ private boolean shouldPerformActionOnNetworkConsumingRun(final PipelineRun run, private void performActionOnNetworkConsumingRun(final PipelineRun run, final NetworkConsumingRunAction action, final double bandwidth, - final List> pipelinesToNotify, - final List runsToUpdate) { - log.info(messageHelper.getMessage(MessageConstants.INFO_RUN_IDLE_ACTION, run.getPodId(), - bandwidth, action)); + final List> runsToNotify, + final List runsToUpdateNotificationTime) { + log.info(messageHelper.getMessage(MessageConstants.INFO_RUN_HIGH_NETWORK_CONSUMPTION_ACTION, + run.getPodId(), bandwidth, action.name())); switch (action) { case LIMIT_BANDWIDTH: // TODO break; + case NOTIFY: default: - performHighNetworkConsumingNotify(run, bandwidth, pipelinesToNotify); + performHighNetworkConsumingNotify(run, bandwidth, runsToNotify, runsToUpdateNotificationTime); + break; } - runsToUpdate.add(run); } - private void performHighNetworkConsumingNotify(PipelineRun run, double networkBandwidthLevel, - List> pipelinesToNotify) { + private void performHighNetworkConsumingNotify(final PipelineRun run, final double networkBandwidthLevel, + final List> pipelinesToNotify, + final List runsToUpdateNotificationTime) { run.setLastNetworkConsumptionNotificationTime(DateUtils.nowUTC()); pipelinesToNotify.add(new ImmutablePair<>(run, networkBandwidthLevel)); + runsToUpdateNotificationTime.add(run); } private void performStop(final PipelineRun run, diff --git a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java index 993db5e076..4d9b1c0b98 100644 --- a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java @@ -508,8 +508,11 @@ private void removeUtilizationLevelTags(final PipelineRun run) { final String suffix = preferenceManager.getPreference(SystemPreferences.SYSTEM_RUN_TAG_DATE_SUFFIX); Stream.of(ResourceMonitoringManager.UTILIZATION_LEVEL_LOW, ResourceMonitoringManager.UTILIZATION_LEVEL_HIGH, + ResourceMonitoringManager.NETWORK_CONSUMING_LEVEL_HIGH, ResourceMonitoringManager.UTILIZATION_LEVEL_LOW + suffix, - ResourceMonitoringManager.UTILIZATION_LEVEL_HIGH + suffix) + ResourceMonitoringManager.UTILIZATION_LEVEL_HIGH + suffix, + ResourceMonitoringManager.NETWORK_CONSUMING_LEVEL_HIGH + suffix, + PipelineRunManager.NETWORK_LIMIT + suffix) .filter(run::hasTag) .forEach(run::removeTag); } diff --git a/api/src/main/java/com/epam/pipeline/manager/notification/NotificationManager.java b/api/src/main/java/com/epam/pipeline/manager/notification/NotificationManager.java index 1614d82ab1..2d1d125cac 100644 --- a/api/src/main/java/com/epam/pipeline/manager/notification/NotificationManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/notification/NotificationManager.java @@ -368,7 +368,6 @@ public void notifyHighNetworkConsumingRuns(final List> final Map runParametersFilters = parseRunExcludeParams(); final List> filtered = pipelineNetworkBandwidthPairs.stream() - .filter(pair -> shouldNotifyHighNetworkConsumingRun(pair.getLeft().getId(), type, settings)) .filter(pair -> noneMatchExcludedInstanceType(pair.getLeft(), instanceTypesToExclude)) .filter(pair -> !matchExcludeRunParameters(pair.getLeft(), runParametersFilters)) .collect(Collectors.toList()); @@ -377,13 +376,6 @@ public void notifyHighNetworkConsumingRuns(final List> pair.getRight(), bandwidthLimit, type)) .collect(Collectors.toList()); saveNotifications(messages); - - if (NotificationType.HIGH_CONSUMED_NETWORK_BANDWIDTH.equals(type)) { - final List runIds = filtered.stream() - .map(pair -> pair.getLeft().getId()).collect(Collectors.toList()); - monitoringNotificationDao.updateNotificationTimestamp(runIds, - NotificationType.HIGH_CONSUMED_NETWORK_BANDWIDTH); - } } private NotificationMessage buildMessageForHighNetworkConsumingRun(final NotificationSettings settings, @@ -899,15 +891,6 @@ private boolean shouldNotifyIdleRun(final Long runId, final NotificationType not return shouldNotify(runId, notificationSettings); } - private boolean shouldNotifyHighNetworkConsumingRun(final Long runId, - final NotificationType notificationType, - final NotificationSettings notificationSettings) { - if (!NotificationType.HIGH_CONSUMED_NETWORK_BANDWIDTH.equals(notificationType)) { - return true; - } - return shouldNotify(runId, notificationSettings); - } - private Map parseRunExcludeParams() { final Map excludeParams = preferenceManager.getPreference( SystemPreferences.SYSTEM_NOTIFICATIONS_EXCLUDE_PARAMS); diff --git a/api/src/main/resources/messages.properties b/api/src/main/resources/messages.properties index 0304ccf2f6..6063e00403 100644 --- a/api/src/main/resources/messages.properties +++ b/api/src/main/resources/messages.properties @@ -219,6 +219,7 @@ debug.run.not.idled=Run {0} has CPU usage rate: {1} considered as not idled, tag debug.run.has.not.node.name=Pipeline with id: {0} has not node name. debug.memory.metrics.received=Memory and disk metrics received ''{0}'' info.run.high.network.consumption.notify=Pipeline Run {0} is high network consuming: bandwidth: {1}, notification will be sent +info.run.high.network.consumption.action=Pipeline Run {0} is high network consuming: bandwidth: {1}, action: {2} will be performed debug.network.run.metrics.received=Network metrics ''{0}'' debug.run.not.network.consuming=Run {0} has network bandwidth: {1} considered as not high consuming, tag will be cleared. debug.run.network.consuming.disabled=SystemPreference system.pod.bandwidth.limit is set to 0, considering it disabled!