From 9d77b533dcc2a72aca1f85d6bc39473fb6672c21 Mon Sep 17 00:00:00 2001 From: Oksana Kolesnikova <83080415+okolesn@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:50:17 +0300 Subject: [PATCH] Issue 3602 pod network bandwidth usage action (#3621) --- api/profiles/dev/application.properties | 2 + .../epam/pipeline/acl/run/RunApiService.java | 5 + .../pipeline/common/MessageConstants.java | 1 + .../pipeline/PipelineRunController.java | 14 ++ .../DockerContainerOperationManager.java | 43 +++++ .../manager/docker/LimitBandwidthCommand.java | 55 ++++++ .../NotificationParameterManager.java | 1 - .../pipeline/BandwidthMonitoringService.java | 36 ++++ .../BandwidthMonitoringServiceCore.java | 73 ++++++++ .../manager/pipeline/PipelineRunManager.java | 26 ++- .../manager/preference/SystemPreferences.java | 7 + api/src/main/resources/messages.properties | 1 + .../resources/test-application.properties | 1 + .../cp-api-srv/config/application.properties | 1 + .../autoscaling/init_multicloud_v1.15.4.sh | 15 ++ scripts/commit-run-scripts/limit_bandwidth.sh | 164 ++++++++++++++++++ 16 files changed, 442 insertions(+), 3 deletions(-) create mode 100644 api/src/main/java/com/epam/pipeline/manager/docker/LimitBandwidthCommand.java create mode 100644 api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringService.java create mode 100644 api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringServiceCore.java create mode 100644 scripts/commit-run-scripts/limit_bandwidth.sh diff --git a/api/profiles/dev/application.properties b/api/profiles/dev/application.properties index c0f9eb36ec..6604abed8e 100644 --- a/api/profiles/dev/application.properties +++ b/api/profiles/dev/application.properties @@ -79,6 +79,8 @@ commit.run.script.starter.url= docker.registry.login.script= container.layers.script.url= container.size.script.url= +limit.run.bandwidth.script.url= + #pause/resume run scripts pause.run.script.url= diff --git a/api/src/main/java/com/epam/pipeline/acl/run/RunApiService.java b/api/src/main/java/com/epam/pipeline/acl/run/RunApiService.java index 1f851fb429..1004d0c0f3 100644 --- a/api/src/main/java/com/epam/pipeline/acl/run/RunApiService.java +++ b/api/src/main/java/com/epam/pipeline/acl/run/RunApiService.java @@ -398,4 +398,9 @@ public void archiveRuns() { public void archiveRuns(final String name, final boolean principal, final Integer days) { archiveRunService.archiveRuns(name, principal, days); } + + @PreAuthorize(ADMIN_ONLY) + public void setLimitBoundary(final Long runId, final Boolean enable, final Integer boundary) { + runManager.setLimitBoundary(runId, enable, boundary); + } } diff --git a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java index 812130d939..2e8137c4ac 100644 --- a/api/src/main/java/com/epam/pipeline/common/MessageConstants.java +++ b/api/src/main/java/com/epam/pipeline/common/MessageConstants.java @@ -150,6 +150,7 @@ public final class MessageConstants { public static final String ERROR_RUN_PIPELINES_COMMIT_FAILED = "error.run.pipeline.commit.failed"; public static final String ERROR_GET_CONTAINER_LAYERS_COUNT_FAILED = "error.container.layers.count.failed"; public static final String ERROR_GET_CONTAINER_SIZE_FAILED = "error.container.size.failed"; + public static final String ERROR_LIMIT_NETWORK_BANDWIDTH_FAILED = "error.limit.network.bandwidth.failed"; public static final String ERROR_CONTAINER_ID_FOR_RUN_NOT_FOUND = "error.container.id.for.run.not.found"; public static final String INFO_EXECUTE_COMMIT_RUN_PIPELINES = "info.execute.ssh.run.pipeline.command"; public static final String ERROR_RUN_PIPELINES_PAUSE_FAILED = "error.run.pipeline.pause.failed"; diff --git a/api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineRunController.java b/api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineRunController.java index 8a709cf13c..015737a7c7 100644 --- a/api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineRunController.java +++ b/api/src/main/java/com/epam/pipeline/controller/pipeline/PipelineRunController.java @@ -620,4 +620,18 @@ public Result archiveRunsByOwner(@RequestParam final String ownerSid, runApiService.archiveRuns(ownerSid, principal, days); return Result.success(true); } + + @PostMapping("/run/{runId}/network/limit") + @ApiOperation( + value = "Set limit boundary", + notes = "Sets a special tag for a run based on boundary param: NETWORK_LIMIT: (Bytes/s) " + + "in case of enable = true, otherwise removes the tag.", + produces = MediaType.APPLICATION_JSON_VALUE) + @ApiResponses(value = {@ApiResponse(code = HTTP_STATUS_OK, message = API_STATUS_DESCRIPTION)}) + public Result setLimitBoundary(@PathVariable(value = RUN_ID) final Long runId, + @RequestParam(defaultValue = "true") final Boolean enable, + @RequestParam(required = false) final Integer boundary) { + runApiService.setLimitBoundary(runId, enable, boundary); + return Result.success(); + } } diff --git a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java index 8c722c04a6..993db5e076 100644 --- a/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/docker/DockerContainerOperationManager.java @@ -89,6 +89,8 @@ public class DockerContainerOperationManager { private static final String GLOBAL_KNOWN_HOSTS_FILE = "GlobalKnownHostsFile=/dev/null"; private static final String USER_KNOWN_HOSTS_FILE = "UserKnownHostsFile=/dev/null"; private static final String COMMIT_COMMAND_DESCRIPTION = "ssh session to commit pipeline run"; + private static final String LIMIT_NETWORK_BANDWIDTH_COMMAND_DESCRIPTION = + "ssh session to limit pipeline run network bandwidth"; private static final String CONTAINER_LAYERS_COUNT_COMMAND_DESCRIPTION = "ssh session to get docker container layers count"; private static final String PAUSE_COMMAND_DESCRIPTION = "Error is occurred during to pause pipeline run"; @@ -103,6 +105,7 @@ public class DockerContainerOperationManager { private static final String CP_NODE_SSH_PORT = "CP_NODE_SSH_PORT"; public static final String PAUSE_RUN_TASK = "PausePipelineRun"; + private static final int BYTES_IN_KB = 1024; @Autowired private PipelineRunManager runManager; @@ -158,6 +161,9 @@ public class DockerContainerOperationManager { @Value("${pause.run.script.url}") private String pauseRunScriptUrl; + @Value("${limit.run.bandwidth.script.url}") + private String limitRunBandwidthScriptUrl; + private Lock resumeLock = new ReentrantLock(); public PipelineRun commitContainer(PipelineRun run, DockerRegistry registry, @@ -302,6 +308,43 @@ public long getContainerSize(final PipelineRun run) { return size; } + public void limitNetworkBandwidth(final PipelineRun run, final Integer boundary, final Boolean enable) { + final String containerId = kubernetesManager.getContainerIdFromKubernetesPod(run.getPodId(), + run.getActualDockerImage()); + try { + Assert.notNull(containerId, + messageHelper.getMessage(MessageConstants.ERROR_CONTAINER_ID_FOR_RUN_NOT_FOUND, run.getId())); + + final String apiToken = authManager.issueTokenForCurrentUser(null).getToken(); + + final int boundaryKBitsPerSec = enable ? boundary * 8 / BYTES_IN_KB : 0; + final String limitNetworkCommand = LimitBandwidthCommand.builder() + .runScriptUrl(limitRunBandwidthScriptUrl) + .runId(String.valueOf(run.getId())) + .api(preferenceManager.getPreference(SystemPreferences.BASE_API_HOST)) + .apiToken(apiToken) + .containerId(containerId) + .enable(String.valueOf(enable)) + .uploadRate(String.valueOf(boundaryKBitsPerSec)) + .downloadRate(String.valueOf(boundaryKBitsPerSec)) + .build() + .getCommand(); + final Process sshConnection = submitCommandViaSSH(run.getInstance().getNodeIP(), limitNetworkCommand, + getSshPort(run)); + final boolean isFinished = sshConnection.waitFor( + preferenceManager.getPreference(SystemPreferences.LIMIT_NETWORK_BANDWIDTH_COMMAND_TIMEOUT), + TimeUnit.SECONDS); + Assert.state(isFinished && sshConnection.exitValue() == 0, + messageHelper.getMessage(MessageConstants.ERROR_LIMIT_NETWORK_BANDWIDTH_FAILED, run.getId())); + } catch (IllegalStateException | IllegalArgumentException | IOException e) { + log.error(e.getMessage()); + throw new CmdExecutionException(LIMIT_NETWORK_BANDWIDTH_COMMAND_DESCRIPTION, e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CmdExecutionException(LIMIT_NETWORK_BANDWIDTH_COMMAND_DESCRIPTION, e); + } + } + @Async("pauseRunExecutor") public void pauseRun(final PipelineRun run, final boolean rerunPause) { try { diff --git a/api/src/main/java/com/epam/pipeline/manager/docker/LimitBandwidthCommand.java b/api/src/main/java/com/epam/pipeline/manager/docker/LimitBandwidthCommand.java new file mode 100644 index 0000000000..f0f10d18ea --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/docker/LimitBandwidthCommand.java @@ -0,0 +1,55 @@ +/* + * Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.manager.docker; + +import lombok.Builder; + +import java.util.ArrayList; +import java.util.List; + +@Builder +public class LimitBandwidthCommand extends AbstractDockerCommand { + private static final String COMMAND_TEMPLATE = "curl -k -s \"%s\" | sudo -E /bin/bash --login /dev/stdin %s"; + + private final String runId; + private final String api; + private final String apiToken; + private final String containerId; + private final String enable; + private final String uploadRate; + private final String downloadRate; + + private final String runScriptUrl; + + @Override + public String getCommand() { + return getDockerCommand(COMMAND_TEMPLATE, runScriptUrl); + } + + @Override + protected List buildCommandArguments() { + final List command = new ArrayList<>(); + command.add(runId); + command.add(api); + command.add(apiToken); + command.add(containerId); + command.add(enable); + command.add(uploadRate); + command.add(downloadRate); + return command; + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/notification/NotificationParameterManager.java b/api/src/main/java/com/epam/pipeline/manager/notification/NotificationParameterManager.java index 4bfbed7a7b..61a738062a 100644 --- a/api/src/main/java/com/epam/pipeline/manager/notification/NotificationParameterManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/notification/NotificationParameterManager.java @@ -38,7 +38,6 @@ public class NotificationParameterManager { private static final double PERCENT = 100.0; - private static final int BYTES_IN_KB = 1024; private final JsonMapper jsonMapper; diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringService.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringService.java new file mode 100644 index 0000000000..8c3f3fdf06 --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringService.java @@ -0,0 +1,36 @@ +/* + * Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.manager.pipeline; + +import com.epam.pipeline.manager.preference.SystemPreferences; +import com.epam.pipeline.manager.scheduling.AbstractSchedulingManager; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; + +@Service +@RequiredArgsConstructor +public class BandwidthMonitoringService extends AbstractSchedulingManager { + private final BandwidthMonitoringServiceCore core; + + @PostConstruct + public void init() { + scheduleFixedDelaySecured(core::monitor, SystemPreferences.SYSTEM_POD_BANDWIDTH_MONITOR_DELAY, + "BandwidthMonitor"); + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringServiceCore.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringServiceCore.java new file mode 100644 index 0000000000..203f4faffc --- /dev/null +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/BandwidthMonitoringServiceCore.java @@ -0,0 +1,73 @@ +/* + * Copyright 2024 EPAM Systems, Inc. (https://www.epam.com/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.pipeline.manager.pipeline; + +import com.epam.pipeline.controller.vo.TagsVO; +import com.epam.pipeline.entity.pipeline.PipelineRun; +import com.epam.pipeline.entity.utils.DateUtils; +import com.epam.pipeline.manager.docker.DockerContainerOperationManager; +import com.epam.pipeline.manager.preference.PreferenceManager; +import com.epam.pipeline.manager.preference.SystemPreferences; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.SchedulerLock; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +import static com.epam.pipeline.manager.pipeline.PipelineRunManager.NETWORK_LIMIT; + +@Service +@Slf4j +@RequiredArgsConstructor +public class BandwidthMonitoringServiceCore { + private final PipelineRunManager pipelineRunManager; + private final DockerContainerOperationManager dockerContainerOperationManager; + private final PreferenceManager preferenceManager; + + @SchedulerLock(name = "BandwidthMonitoringService_monitor", lockAtMostForString = "PT10M") + public void monitor() { + final List runs = pipelineRunManager.loadRunningPipelineRuns(); + final String networkLimitDateTag = getNetworkLimitDateTag(); + for (PipelineRun run: runs) { + final Map tags = run.getTags(); + if (shouldSetLimit(tags)) { + final int boundary = Integer.parseInt(tags.get(NETWORK_LIMIT)); + dockerContainerOperationManager.limitNetworkBandwidth(run, boundary, true); + run.addTag(networkLimitDateTag, DateUtils.nowUTCStr()); + } else if (shouldCleanLimit(tags)) { + dockerContainerOperationManager.limitNetworkBandwidth(run, 0, false); + run.removeTag(networkLimitDateTag); + } + pipelineRunManager.updateTags(run.getId(), new TagsVO(run.getTags()), true); + } + } + + private boolean shouldSetLimit(final Map tags) { + return tags.containsKey(NETWORK_LIMIT) && !tags.containsKey(getNetworkLimitDateTag()); + } + + private boolean shouldCleanLimit(final Map tags) { + return !tags.containsKey(NETWORK_LIMIT) && tags.containsKey(getNetworkLimitDateTag()); + } + + private String getNetworkLimitDateTag() { + final String suffix = preferenceManager.getPreference(SystemPreferences.SYSTEM_RUN_TAG_DATE_SUFFIX); + return String.format("%s_%s", NETWORK_LIMIT, suffix); + } +} diff --git a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java index a868dfa48d..98d715d290 100644 --- a/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java +++ b/api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunManager.java @@ -151,12 +151,14 @@ public class PipelineRunManager { private static final int DIVIDER_TO_GB = 1024 * 1024 * 1024; private static final int USER_PRICE_SCALE = 2; private static final int BILLING_PRICE_SCALE = 5; - public static final String CP_CAP_LIMIT_MOUNTS = "CP_CAP_LIMIT_MOUNTS"; private static final String LIMIT_MOUNTS_NONE = "none"; private static final String CP_REPORT_RUN_STATUS = "CP_REPORT_RUN_STATUS"; private static final String CP_REPORT_RUN_PROCESSED_DATE = "CP_REPORT_RUN_PROCESSED_DATE"; private static final String CP_GPU_COUNT = "CP_GPU_COUNT"; + public static final String CP_CAP_LIMIT_MOUNTS = "CP_CAP_LIMIT_MOUNTS"; + public static final String NETWORK_LIMIT = "NETWORK_LIMIT"; + @Autowired private PipelineRunDao pipelineRunDao; @@ -371,7 +373,6 @@ public void prolongIdleRun(Long runId) { updateProlongIdleRunAndLastIdleNotificationTime(run); } - /** * Internal method for creating a pipeline run, * it assumes that ACL filtering was already applied to input arguments @@ -1340,6 +1341,27 @@ public RunChartInfo loadActiveRunsCharts(final RunChartFilterVO filter) { .build(); } + @Transactional(propagation = Propagation.REQUIRED) + public void setLimitBoundary(final Long runId, final Boolean enable, final Integer boundary) { + Assert.isTrue(!enable || boundary != null, "Boundary value should be specified to limit network bandwidth"); + final PipelineRun run = pipelineRunDao.loadPipelineRun(runId); + Assert.notNull(run, + messageHelper.getMessage(MessageConstants.ERROR_PIPELINE_NOT_FOUND, runId)); + final Map tags = new HashMap<>(MapUtils.emptyIfNull(run.getTags())); + if (enable) { + tags.put(NETWORK_LIMIT, String.valueOf(boundary)); + // Clear tag with timestamp of applied limit, to re-enable it with a new bound. + final String timestampTagSuffix = preferenceManager.getPreference( + SystemPreferences.SYSTEM_RUN_TAG_DATE_SUFFIX + ); + tags.remove(String.format("%s_%s", NETWORK_LIMIT, timestampTagSuffix)); + } else { + tags.remove(NETWORK_LIMIT); + } + run.setTags(tags); + pipelineRunDao.updateRunTags(run); + } + private int getTotalSize(final List disks) { return (int) disks.stream().mapToLong(InstanceDisk::getSize).sum(); } diff --git a/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java b/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java index 6885feb4ea..9d8b399753 100644 --- a/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java +++ b/api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java @@ -1166,6 +1166,7 @@ public class SystemPreferences { public static final BooleanPreference SYSTEM_NOTIFICATIONS_ENABLE = new BooleanPreference( "system.notifications.enable", false, SYSTEM_GROUP, pass); + //in bytes per second public static final DoublePreference SYSTEM_POD_BANDWIDTH_LIMIT = new DoublePreference( "system.pod.bandwidth.limit", 0.0, SYSTEM_GROUP, isGreaterThanOrEquals(0.0f)); @@ -1187,6 +1188,12 @@ public class SystemPreferences { "system.archive.run.owners.chunk.size", 100, SYSTEM_GROUP, isGreaterThan(0)); + public static final IntPreference SYSTEM_POD_BANDWIDTH_MONITOR_DELAY = new IntPreference( + "system.pod.bandwidth.monitor.delay", 30000, SYSTEM_GROUP, pass); + + public static final IntPreference LIMIT_NETWORK_BANDWIDTH_COMMAND_TIMEOUT = new IntPreference( + "limit.network.bandwidth.command.timeout", 600, SYSTEM_GROUP, isGreaterThan(0)); + // FireCloud Integration public static final ObjectPreference> FIRECLOUD_SCOPES = new ObjectPreference<>( "firecloud.api.scopes", null, new TypeReference>() {}, FIRECLOUD_GROUP, diff --git a/api/src/main/resources/messages.properties b/api/src/main/resources/messages.properties index 452c9d2244..0304ccf2f6 100644 --- a/api/src/main/resources/messages.properties +++ b/api/src/main/resources/messages.properties @@ -87,6 +87,7 @@ error.run.pipeline.commit.failed=Process of the commit of the run ''{0}'' via ss error.container.id.for.run.not.found=Container id for run ''{0}'' not found! error.container.layers.count.failed=Get container layers count for run ''{0}'' failed! error.container.size.failed=Get container size for run ''{0}'' failed! +error.limit.network.bandwidth.failed=Limit network bandwidth for run ''{0}'' failed! error.pipeline.run.finished=Pipeline run with id: ''{0}'' already finished. error.pipeline.run.not.stopped=Pipeline run with id: ''{0}'' not stopped. error.pipeline.run.not.running=Run {0} shall be in 'RUNNING' status to perform this operation. diff --git a/api/src/test/resources/test-application.properties b/api/src/test/resources/test-application.properties index 82aabdaa28..fe22aa02b7 100644 --- a/api/src/test/resources/test-application.properties +++ b/api/src/test/resources/test-application.properties @@ -49,6 +49,7 @@ commit.run.scripts.root.url= commit.run.script.starter.url= container.layers.script.url= container.size.script.url= +limit.run.bandwidth.script.url= #pause/resume run scripts pause.run.script.url= diff --git a/deploy/docker/cp-api-srv/config/application.properties b/deploy/docker/cp-api-srv/config/application.properties index ff97968f9c..4034f3c0b3 100644 --- a/deploy/docker/cp-api-srv/config/application.properties +++ b/deploy/docker/cp-api-srv/config/application.properties @@ -150,6 +150,7 @@ commit.run.scripts.root.url=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INT commit.run.script.starter.url=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/commit-run-scripts/commit_run_starter.sh container.layers.script.url=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/commit-run-scripts/container_layers_count.sh container.size.script.url=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/commit-run-scripts/container_size.sh +limit.run.bandwidth.script.url=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/commit-run-scripts/limit_bandwidth.sh launch.script.url.linux=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/launch.sh init.script.url.linux=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/init.sh launch.script.url.windows=https://${CP_API_SRV_INTERNAL_HOST}:${CP_API_SRV_INTERNAL_PORT}/pipeline/launch.py diff --git a/scripts/autoscaling/init_multicloud_v1.15.4.sh b/scripts/autoscaling/init_multicloud_v1.15.4.sh index f70572736d..2a4aa30a1a 100644 --- a/scripts/autoscaling/init_multicloud_v1.15.4.sh +++ b/scripts/autoscaling/init_multicloud_v1.15.4.sh @@ -252,6 +252,21 @@ if [ $? -ne 0 ]; then _DOCKER_SYS_IMGS="/opt/docker-system-images" fi + +cd /bin && \ +wget https://github.com/magnific0/wondershaper/archive/refs/heads/master.zip -O wondershaper.zip && \ +unzip wondershaper.zip && \ +mv wondershaper-master wondershaper && \ +rm -rf wondershaper.zip && \ +cd wondershaper && \ +sudo make install +if check_installed "wondershaper"; then + echo "[INFO] Wondershaper was installed successfully." +else + echo "[WARN] Wondershaper installation failed." +fi + + mkdir -p /etc/docker if [[ $FS_TYPE == "ext4" ]]; then diff --git a/scripts/commit-run-scripts/limit_bandwidth.sh b/scripts/commit-run-scripts/limit_bandwidth.sh new file mode 100644 index 0000000000..29f7fe7231 --- /dev/null +++ b/scripts/commit-run-scripts/limit_bandwidth.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright 2017-2019 EPAM Systems, Inc. (https://www.epam.com/) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function call_api() { + _API="$1" + _API_TOKEN="$2" + _API_METHOD="$3" + _HTTP_METHOD="$4" + _HTTP_BODY="$5" + if [[ "$_HTTP_BODY" ]] + then + curl -f -s -k -X "$_HTTP_METHOD" \ + --header 'Accept: application/json' \ + --header 'Authorization: Bearer '"$_API_TOKEN" \ + --header 'Content-Type: application/json' \ + --data "$_HTTP_BODY" \ + "$_API$_API_METHOD" + else + curl -f -s -k -X "$_HTTP_METHOD" \ + --header 'Accept: application/json' \ + --header 'Authorization: Bearer '"$_API_TOKEN" \ + --header 'Content-Type: application/json' \ + "$_API$_API_METHOD" + fi +} + +ERROR_LOG_LEVEL="ERROR" +INFO_LOG_LEVEL="INFO" +DEBUG_LOG_LEVEL="DEBUG" + +function pipe_api_log() { + _MESSAGE="$1" + _STATUS="$2" + if [ "$RUN_ID" ] && [ "$LOG_TASK" ]; then + if [ "$_STATUS" == "$ERROR_LOG_LEVEL" ]; then + STATUS="FAILURE" + else + STATUS="RUNNING" + fi + call_api "$API" "$API_TOKEN" "run/$RUN_ID/log" "POST" '{ + "date": "'"$(get_current_date)"'", + "logText": "'"$_MESSAGE"'", + "runId": '"$RUN_ID"', + "status": "'"$STATUS"'", + "taskName": "'"$LOG_TASK"'" + }' + fi +} + +function get_current_date() { + date '+%Y-%m-%d %H:%M:%S.%N' | cut -b1-23 +} + +function pipe_log_info() { + _MESSAGE="$1" + pipe_log "$_MESSAGE" "$INFO_LOG_LEVEL" +} + +function pipe_log_error() { + _MESSAGE="$1" + pipe_log "$_MESSAGE" "$ERROR_LOG_LEVEL" +} + +function pipe_log() { + _MESSAGE="$1" + _STATUS="$2" + echo "$(get_current_date): [$_STATUS] $_MESSAGE" + if [[ "$DEBUG" ]] || [[ "$_STATUS" != "$DEBUG_LOG_LEVEL" ]] + then + pipe_api_log "$_MESSAGE" "$_STATUS" + fi +} + +export LOG_TASK="LimitNetworkBandwidth" + +export RUN_ID="$1" +export API="$2" +export API_TOKEN="$3" +container_id="$4" +enable="$5" +upload_rate="$6" +download_rate="$7" + +COMMAND="/bin/wondershaper/wondershaper" + +if [ -f "$COMMAND" ]; then + pipe_log_info "[INFO] ${COMMAND} exists. Proceeding." +else + pipe_log_info "[WARN] No ${COMMAND} found. Trying to install." + + cd /bin && \ + wget https://github.com/magnific0/wondershaper/archive/refs/heads/master.zip -O wondershaper.zip && \ + unzip wondershaper.zip && \ + mv wondershaper-master wondershaper && \ + rm -rf wondershaper.zip && \ + cd wondershaper && \ + sudo make install + + if [ -f "$COMMAND" ]; then + pipe_log_info "[INFO] ${COMMAND} installed successfully. Proceeding." + else + pipe_log_error "[ERROR] Fail to install wondershaper. Exiting." + exit 1 + fi +fi + +interfaces=($(sudo docker exec "$container_id" ls /sys/class/net)) +if [ "$?" -eq 0 ]; then + pipe_log_info "[INFO] Network bandwidth limitation will be applied." +else + pipe_log_error "[ERROR] No interfaces found. Network bandwidth limitation won't be applied." + exit 1 +fi + +for i in "${interfaces[@]}"; do + if [ "$i" == "lo" ] || [ -z "$i" ]; then + continue + fi + + link_num=$(sudo docker exec $container_id cat /sys/class/net/$i/iflink | tr -d '\r') + interface_id=$(sudo ip ad | grep "^$link_num:" | awk -F ': ' '{print $2}' | awk -F '@' '{print $1}') + if [ ! -z "$interface_id" ]; then + if [ $enable == "true" ]; then + pipe_log_info "[INFO] Clearing the old limit for run network interface $i and its node network interface $interface_id, if it exists" + "$COMMAND" -c -a "$interface_id" + pipe_log_info "[INFO] Enabling limit for run network interface $i and its node network interface $interface_id" + # download_rate and upload_rate in kilobits per second + result=$("$COMMAND" -a "$interface_id" -d "$download_rate" -u "$upload_rate") + if [ "$?" -ne 0 ]; then + pipe_log_error "[WARN] Failed to apply bandwidth limiting" + pipe_log_error "[WARN] ${result}" + exit 1 + else + pipe_log_info "[INFO] Bandwidth limiting -d $download_rate -u $upload_rate (Kbit/sec) applied successfully" + fi + else + pipe_log_info "[INFO] Clearing limit for run network interface $i and its node network interface $interface_id" + result=$("$COMMAND" -c -a "$interface_id") + result_code=$? + if [ "$result_code" -ne 0 ] && [ "$result_code" -ne 2 ]; then + pipe_log_error "[WARN] Failed to clear bandwidth limitation" + pipe_log_error "[WARN] Output: ${result}" + exit 1 + else + pipe_log_info "[INFO] Bandwidth limitation cleared" + fi + fi + else + pipe_log_info "[WARN] Interface id wasn't found for "$i" interface" + fi +done