Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix PortfolioMetricsUpdateTask refreshing too many project metrics concurrently #919

Merged
merged 1 commit into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import net.javacrumbs.shedlock.core.LockConfiguration;
import net.javacrumbs.shedlock.core.LockExtender;
import net.javacrumbs.shedlock.core.LockingTaskExecutor;
import org.apache.commons.collections4.ListUtils;
import org.dependencytrack.event.CallbackEvent;
import org.dependencytrack.event.PortfolioMetricsUpdateEvent;
import org.dependencytrack.event.ProjectMetricsUpdateEvent;
Expand All @@ -36,6 +35,8 @@
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
Expand All @@ -45,7 +46,6 @@
import static org.dependencytrack.util.LockProvider.isTaskLockToBeExtended;
import static org.dependencytrack.util.TaskUtil.getLockConfigForTask;


/**
* A {@link Subscriber} task that updates portfolio metrics.
*
Expand Down Expand Up @@ -87,7 +87,7 @@
}

private static void refreshProjectMetrics() throws Exception {
try (final var qm = new QueryManager().withL2CacheDisabled()) {
try (final var qm = new QueryManager()) {
final PersistenceManager pm = qm.getPersistenceManager();

LOGGER.debug("Fetching first " + BATCH_SIZE + " projects");
Expand All @@ -100,7 +100,7 @@
final long lastId = activeProjects.get(activeProjects.size() - 1).id();

// Distribute the batch across at most MAX_CONCURRENCY events, and process them asynchronously.
final List<List<ProjectProjection>> partitions = ListUtils.partition(activeProjects, MAX_CONCURRENCY);
final List<List<ProjectProjection>> partitions = partition(activeProjects, MAX_CONCURRENCY);
final var countDownLatch = new CountDownLatch(partitions.size());

for (final List<ProjectProjection> partition : partitions) {
Expand Down Expand Up @@ -145,6 +145,9 @@
}
}

public record ProjectProjection(long id, UUID uuid) {
}

private static List<ProjectProjection> fetchNextActiveProjectsPage(final PersistenceManager pm, final Long lastId) throws Exception {
try (final Query<Project> query = pm.newQuery(Project.class)) {
if (lastId == null) {
Expand All @@ -160,7 +163,24 @@
}
}

public record ProjectProjection(long id, UUID uuid) {
static <T> List<List<T>> partition(final List<T> list, int numPartitions) {
if (list == null || list.isEmpty()) {
return Collections.emptyList();
}

final int listSize = list.size();
final var partitions = new ArrayList<List<T>>(numPartitions);
int partitionSize = (int) Math.ceil((double) listSize / numPartitions);

int i = 0, elementsLeft = listSize;

Check warning on line 175 in src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java#L175

Use one line for each declaration, it enhances code readability.
while (i < listSize && numPartitions != 0) {
partitions.add(list.subList(i, i + partitionSize));
i = i + partitionSize;
elementsLeft = elementsLeft - partitionSize;
partitionSize = (int) Math.ceil((double) elementsLeft / --numPartitions);

Check warning on line 180 in src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

src/main/java/org/dependencytrack/tasks/metrics/PortfolioMetricsUpdateTask.java#L180

Avoid reassigning parameters such as 'numPartitions'
}

return partitions;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,12 @@
import org.junit.Test;

import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;
import static org.dependencytrack.tasks.metrics.PortfolioMetricsUpdateTask.partition;

@NotThreadSafe
public class PortfolioMetricsUpdateTaskTest extends AbstractMetricsUpdateTaskTest {
Expand Down Expand Up @@ -363,4 +365,36 @@ public void testUpdateMetricsPolicyViolations() {
assertThat(componentSuppressed.getLastInheritedRiskScore()).isZero();
}

@Test
public void testPartitionWithNull() {
final List<Integer> list = null;
final List<List<Integer>> partitions = partition(list, 4);
assertThat(partitions).isEmpty();
}

@Test
public void testPartitionWithEmptyList() {
final List<Integer> list = Collections.emptyList();
final List<List<Integer>> partitions = partition(list, 4);
assertThat(partitions).isEmpty();
}

@Test
public void testPartitionWithSmallList() {
final List<Integer> list = List.of(1, 2);
final List<List<Integer>> partitions = partition(list, 4);
assertThat(partitions).hasSize(2);
}

@Test
public void testPartitionWithUnevenSizeList() {
final List<Integer> list = List.of(1, 2, 3, 4, 5);
final List<List<Integer>> partitions = partition(list, 4);
assertThat(partitions).satisfiesExactlyInAnyOrder(
partition -> assertThat(partition).hasSize(2),
partition -> assertThat(partition).hasSize(1),
partition -> assertThat(partition).hasSize(1),
partition -> assertThat(partition).hasSize(1));
}

}
Loading