Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 3573 Cleanup and tests #3594

Merged
merged 1 commit into from
Jul 8, 2024
Merged

Issue 3573 Cleanup and tests #3594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/src/main/java/com/epam/pipeline/acl/run/RunApiService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ public void prolongIdleRun(Long runId) {
@PreAuthorize(RUN_ID_READ)
public CommitRunConditions getCommitRunCheckResult(final Long runId) {
return CommitRunConditions.builder()
.containerSize(pipelineRunDockerOperationManager.getContainerSize(runId))
.containerSize(pipelineRunDockerOperationManager.checkContainerSizeForCommit(runId))
.enoughSpace(pipelineRunDockerOperationManager.checkFreeSpaceAvailable(runId))
.build();
}
Expand Down
8 changes: 8 additions & 0 deletions api/src/main/java/com/epam/pipeline/entity/utils/TwoBoundaryLimit.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,12 @@
public class TwoBoundaryLimit {
private Long soft;
private Long hard;

public boolean isSoftLimitDefined() {
return soft != null && soft > 0;
}

public boolean isHardLimitDefined() {
return hard != null && hard > 0;
}
}
32 changes: 19 additions & 13 deletions api/src/main/java/com/epam/pipeline/manager/pipeline/PipelineRunDockerOperationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,11 @@ public Long getContainerLayers(final Long id) {
return dockerContainerOperationManager.getContainerLayers(pipelineRun);
}

public ConditionCheck<Long> getContainerSize(final Long id) {
public ConditionCheck<Long> checkContainerSizeForCommit(final Long id) {
final PipelineRun pipelineRun = pipelineRunDao.loadPipelineRun(id);
final long containerSize = dockerContainerOperationManager.getContainerSize(pipelineRun);
final TwoBoundaryLimit limits = preferenceManager.getPreference(SystemPreferences.COMMIT_TOOL_SIZE_LIMITS);
final Pair<ConditionCheck.Result, String> containerSizeCheck = getCommitRunCheckStatus(containerSize, limits);
final Pair<ConditionCheck.Result, String> containerSizeCheck = mapContainerSizeOnLimits(containerSize, limits);
return ConditionCheck.<Long>builder()
.result(containerSizeCheck.getKey())
.message(containerSizeCheck.getValue())
Expand Down Expand Up @@ -224,6 +224,23 @@ public void rerunPauseAndResume() {
ListUtils.emptyIfNull(resumingRuns).forEach(this::rerunResumeRun);
}

static Pair<ConditionCheck.Result, String> mapContainerSizeOnLimits(
final long containerSize, final TwoBoundaryLimit containerSizeLimits) {
if (containerSizeLimits == null) {
return Pair.create(ConditionCheck.Result.OK, null);
}

if (containerSizeLimits.isHardLimitDefined() && containerSize >= containerSizeLimits.getHard()) {
return Pair.create(ConditionCheck.Result.FAIL, "Container is too big to commit!");
} else {
if (containerSizeLimits.isSoftLimitDefined() && containerSize >= containerSizeLimits.getSoft()) {
return Pair.create(ConditionCheck.Result.WARN,
"Container image size may be quite big! Are you sure you want to commit?");
}
}
return Pair.create(ConditionCheck.Result.OK, null);
}

@SuppressWarnings("PMD.AvoidCatchingGenericException")
private void rerunPauseRun(final PipelineRun run) {
try {
Expand Down Expand Up @@ -315,15 +332,4 @@ private long getDiskSpaceAvailable(final PipelineRun pipelineRun) {
return Long.MAX_VALUE;
}
}

private static Pair<ConditionCheck.Result, String> getCommitRunCheckStatus(
final long containerSize, final TwoBoundaryLimit containerSizeLimits) {
if (containerSizeLimits == null || containerSize <= containerSizeLimits.getSoft()) {
return Pair.create(ConditionCheck.Result.OK, null);
} else if (containerSize <= containerSizeLimits.getHard()) {
return Pair.create(ConditionCheck.Result.WARN, "Container image size may be quite big!");
} else {
return Pair.create(ConditionCheck.Result.FAIL, "Container is too big for commit.");
}
}
}
11 changes: 7 additions & 4 deletions api/src/main/java/com/epam/pipeline/manager/preference/SystemPreferences.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,10 @@ public class SystemPreferences {
public static final IntPreference COMMIT_MAX_LAYERS = new IntPreference("commit.max.layers", 127,
COMMIT_GROUP, isGreaterThan(0));
public static final ObjectPreference<TwoBoundaryLimit> COMMIT_TOOL_SIZE_LIMITS = new ObjectPreference<>(
"commit.container.size.limits", null, new TypeReference<TwoBoundaryLimit>() {},
COMMIT_GROUP, isNullOrValidJson(new TypeReference<TwoBoundaryLimit>() {}));
"commit.container.size.limits", TwoBoundaryLimit.builder().soft(0L).hard(0L).build(),
new TypeReference<TwoBoundaryLimit>() {}, COMMIT_GROUP,
isNullOrValidJson(new TypeReference<TwoBoundaryLimit>() {}), true
);

public static final IntPreference GET_CONTAINER_SIZE_TIMEOUT = new IntPreference("get.container.size.timeout", 600,
COMMIT_GROUP, isGreaterThan(0));
Expand Down Expand Up @@ -732,8 +734,9 @@ public class SystemPreferences {
"launch.run.reschedule.enabled", true, LAUNCH_GROUP, pass);

public static final ObjectPreference<TwoBoundaryLimit> RUN_TOOL_SIZE_LIMITS = new ObjectPreference<>(
"launch.tool.size.limits", null, new TypeReference<TwoBoundaryLimit>() {},
LAUNCH_GROUP, isNullOrValidJson(new TypeReference<TwoBoundaryLimit>() {}));
"launch.tool.size.limits", TwoBoundaryLimit.builder().soft(0L).hard(0L).build(),
new TypeReference<TwoBoundaryLimit>() {}, LAUNCH_GROUP,
isNullOrValidJson(new TypeReference<TwoBoundaryLimit>() {}), true);

/**
* Specifies a comma-separated list of environment variables that should be inherited by DIND containers
Expand Down
48 changes: 48 additions & 0 deletions ...c/test/java/com/epam/pipeline/manager/pipeline/PipelineRunDockerOperationManagerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,16 @@
import com.epam.pipeline.entity.pipeline.TaskStatus;
import com.epam.pipeline.entity.pipeline.Tool;
import com.epam.pipeline.entity.pipeline.run.RunStatus;
import com.epam.pipeline.entity.utils.ConditionCheck;
import com.epam.pipeline.entity.utils.TwoBoundaryLimit;
import com.epam.pipeline.manager.ObjectCreatorUtils;
import com.epam.pipeline.manager.cluster.performancemonitoring.UsageMonitoringManager;
import com.epam.pipeline.manager.docker.DockerContainerOperationManager;
import com.epam.pipeline.manager.docker.DockerRegistryManager;
import com.epam.pipeline.manager.preference.PreferenceManager;
import com.epam.pipeline.manager.quota.RunLimitsService;
import org.apache.commons.lang.time.DateUtils;
import org.junit.Assert;
import org.junit.Test;

import java.time.LocalDateTime;
Expand Down Expand Up @@ -156,6 +159,51 @@ public void shouldNotFailResumeIfPauseFailed() {
verify(dockerContainerOperationManager).resumeRun(runToResume, TEST_NAMES);
}

@Test
public void shouldCorrectlyCalculateContainerSizeAgainstLimits() {
TwoBoundaryLimit limit = TwoBoundaryLimit.builder().soft(Long.MAX_VALUE / 2).hard(Long.MAX_VALUE).build();
Assert.assertEquals(
ConditionCheck.Result.OK,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE / 4, limit).getKey()
);

limit = TwoBoundaryLimit.builder().soft(Long.MAX_VALUE / 2).hard(Long.MAX_VALUE).build();
Assert.assertEquals(
ConditionCheck.Result.WARN,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE / 2 + 1, limit).getKey()
);

limit = TwoBoundaryLimit.builder().soft(Long.MAX_VALUE / 2).hard(Long.MAX_VALUE - 1).build();
Assert.assertEquals(
ConditionCheck.Result.FAIL,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE, limit).getKey()
);

limit = TwoBoundaryLimit.builder().hard(Long.MAX_VALUE).build();
Assert.assertEquals(
ConditionCheck.Result.OK,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE - 1, limit).getKey()
);

limit = TwoBoundaryLimit.builder().soft(0L).hard(Long.MAX_VALUE).build();
Assert.assertEquals(
ConditionCheck.Result.OK,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE - 1, limit).getKey()
);

limit = TwoBoundaryLimit.builder().hard(Long.MAX_VALUE / 2).build();
Assert.assertEquals(
ConditionCheck.Result.OK,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE / 4, limit).getKey()
);

limit = TwoBoundaryLimit.builder().hard(Long.MAX_VALUE / 2).build();
Assert.assertEquals(
ConditionCheck.Result.FAIL,
PipelineRunDockerOperationManager.mapContainerSizeOnLimits(Long.MAX_VALUE / 2 + 1, limit).getKey()
);
}

private PipelineRun pipelineRun() {
return ObjectCreatorUtils.createPipelineRun(RUN_ID, null, null, TEST_ID);
}
Expand Down
Loading