Skip to content

Commit

Permalink
Overriding transitive dependencies by root/direct dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
gs-gunjan committed Jul 28, 2023
1 parent 6ae4044 commit 98402f0
Show file tree
Hide file tree
Showing 16 changed files with 402 additions and 65 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@
import org.finos.legend.depot.store.artifacts.api.ParentEvent;
import org.finos.legend.depot.store.artifacts.configuration.ArtifactsRetentionPolicyConfiguration;
import org.finos.legend.depot.store.artifacts.resources.ArtifactsRefreshResource;
import org.finos.legend.depot.store.artifacts.resources.DependenciesResource;
import org.finos.legend.depot.store.artifacts.services.ArtifactsRefreshServiceImpl;
import org.finos.legend.depot.store.artifacts.services.DependencyManager;
import org.finos.legend.depot.store.artifacts.services.ProjectVersionRefreshHandler;
import org.finos.legend.depot.store.notifications.api.NotificationEventHandler;
import org.finos.legend.depot.tracing.api.PrometheusMetricsHandler;
Expand All @@ -45,17 +43,14 @@ public class ArtifactsRefreshModule extends PrivateModule
protected void configure()
{
bind(ArtifactsRefreshResource.class);
bind(DependenciesResource.class);

bind(ArtifactsRefreshService.class).to(ArtifactsRefreshServiceImpl.class);
bind(NotificationEventHandler.class).to(ProjectVersionRefreshHandler.class);
bind(ProjectVersionRefreshHandler.class);
bind(DependencyManager.class);

expose(ArtifactsRefreshService.class);
expose(NotificationEventHandler.class);
expose(ArtifactsRefreshResource.class);
expose(DependenciesResource.class);
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.finos.legend.depot.domain.CoordinateValidator;
import org.finos.legend.depot.domain.api.MetadataEventResponse;
import org.finos.legend.depot.domain.notifications.MetadataNotification;
import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
import org.finos.legend.depot.store.artifacts.configuration.IncludeProjectPropertiesConfiguration;
import org.finos.legend.depot.domain.project.ProjectVersion;
import org.finos.legend.depot.domain.project.ProjectVersionData;
Expand Down Expand Up @@ -82,20 +83,20 @@ public final class ProjectVersionRefreshHandler implements NotificationEventHand
private final List<String> projectPropertiesInScope;
private final List<String> manifestPropertiesInScope;
private final Queue workQueue;
private final DependencyManager dependencyManager;
private final ManageDependenciesService manageDependenciesService;
private final int maximumSnapshotsAllowed;


@Inject
public ProjectVersionRefreshHandler(ManageProjectsService projects, RepositoryServices repositoryServices, Queue workQueue, ArtifactsFilesStore artifacts, IncludeProjectPropertiesConfiguration includePropertyConfig, DependencyManager dependencyManager, @Named("maximumSnapshotsAllowed") int maximumSnapshotsAllowed)
public ProjectVersionRefreshHandler(ManageProjectsService projects, RepositoryServices repositoryServices, Queue workQueue, ArtifactsFilesStore artifacts, IncludeProjectPropertiesConfiguration includePropertyConfig, ManageDependenciesService manageDependenciesService, @Named("maximumSnapshotsAllowed") int maximumSnapshotsAllowed)
{
this.projects = projects;
this.workQueue = workQueue;
this.artifacts = artifacts;
this.repositoryServices = repositoryServices;
this.projectPropertiesInScope = includePropertyConfig != null ? includePropertyConfig.getProperties() : Collections.EMPTY_LIST;
this.manifestPropertiesInScope = includePropertyConfig != null ? includePropertyConfig.getManifestProperties() : Collections.EMPTY_LIST;
this.dependencyManager = dependencyManager;
this.manageDependenciesService = manageDependenciesService;
this.maximumSnapshotsAllowed = maximumSnapshotsAllowed;

try
Expand Down Expand Up @@ -229,8 +230,8 @@ MetadataEventResponse doRefresh(MetadataNotification event)
StoreProjectData project = getProject(event.getGroupId(), event.getArtifactId());
try
{
List<ProjectVersion> newDependencies = this.dependencyManager.calculateDependencies(event.getGroupId(), event.getArtifactId(), event.getVersionId());
this.dependencyManager.validateDependencies(newDependencies, event.getVersionId()).forEach(error -> response.addError(error));
List<ProjectVersion> newDependencies = this.manageDependenciesService.calculateDependencies(event.getGroupId(), event.getArtifactId(), event.getVersionId());
this.manageDependenciesService.validateDependencies(newDependencies, event.getVersionId()).forEach(error -> response.addError(error));
if (!response.hasErrors())
{
LOGGER.info("Processing artifacts for [{}-{}-{}]", event.getGroupId(), event.getArtifactId(), event.getVersionId());
Expand Down Expand Up @@ -281,7 +282,7 @@ private void updateProjectVersionData(StoreProjectData project, String versionId
StoreProjectVersionData storeProjectVersionData = projectVersionData.isPresent() ? projectVersionData.get() : new StoreProjectVersionData(project.getGroupId(), project.getArtifactId(), versionId);
ProjectVersionData versionData = storeProjectVersionData.getVersionData();
versionData.setDependencies(newDependencies);
this.dependencyManager.setProjectDataTransitiveDependencies(storeProjectVersionData);
this.manageDependenciesService.setProjectDataTransitiveDependencies(storeProjectVersionData);

if (this.projectPropertiesInScope != null && !this.projectPropertiesInScope.isEmpty())
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.finos.legend.depot.domain.api.MetadataEventResponse;
import org.finos.legend.depot.domain.api.status.MetadataEventStatus;
import org.finos.legend.depot.domain.notifications.MetadataNotification;
import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
import org.finos.legend.depot.services.dependencies.ManageDependenciesServiceImpl;
import org.finos.legend.depot.store.artifacts.configuration.IncludeProjectPropertiesConfiguration;
import org.finos.legend.depot.domain.project.ProjectVersion;
import org.finos.legend.depot.domain.project.ProjectVersionData;
Expand Down Expand Up @@ -97,9 +99,9 @@ public class TestArtifactsRefreshService extends TestStoreMongo
protected ManageEntitiesService entitiesService = new ManageEntitiesServiceImpl(entitiesStore, projectsService);
protected ArtifactRepository repository = new TestMavenArtifactsRepository();
protected RepositoryServices repositoryServices = new RepositoryServices(repository);
protected DependencyManager dependencyManager = new DependencyManager(projectsService, repositoryServices);
protected ManageDependenciesService manageDependenciesService = new ManageDependenciesServiceImpl(projectsService, repositoryServices);

protected ProjectVersionRefreshHandler versionHandler = new ProjectVersionRefreshHandler(projectsService, repositoryServices, queue, artifacts, new IncludeProjectPropertiesConfiguration(properties, manifestProperties), dependencyManager, 10);
protected ProjectVersionRefreshHandler versionHandler = new ProjectVersionRefreshHandler(projectsService, repositoryServices, queue, artifacts, new IncludeProjectPropertiesConfiguration(properties, manifestProperties), manageDependenciesService, 10);

protected ArtifactsRefreshService artifactsRefreshService = new ArtifactsRefreshServiceImpl(projectsService, repositoryServices, queue);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.finos.legend.depot.domain.api.MetadataEventResponse;
import org.finos.legend.depot.domain.api.status.MetadataEventStatus;
import org.finos.legend.depot.domain.notifications.MetadataNotification;
import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
import org.finos.legend.depot.services.dependencies.ManageDependenciesServiceImpl;
import org.finos.legend.depot.store.artifacts.configuration.IncludeProjectPropertiesConfiguration;
import org.finos.legend.depot.domain.project.StoreProjectData;
import org.finos.legend.depot.domain.project.StoreProjectVersionData;
Expand Down Expand Up @@ -82,9 +84,9 @@ public class TestProjectVersionRefreshHandler extends TestStoreMongo
protected EntityArtifactsProvider entitiesProvider = new EntityProvider();
protected RepositoryServices repositoryServices = mock(RepositoryServices.class);

protected DependencyManager dependencyManager = new DependencyManager(projectsService, repositoryServices);
protected ManageDependenciesService manageDependenciesService = new ManageDependenciesServiceImpl(projectsService, repositoryServices);

protected ProjectVersionRefreshHandler versionHandler = new ProjectVersionRefreshHandler(projectsService, repositoryServices, queue, artifactsStore, new IncludeProjectPropertiesConfiguration(properties, manifestProperties), dependencyManager, 3);
protected ProjectVersionRefreshHandler versionHandler = new ProjectVersionRefreshHandler(projectsService, repositoryServices, queue, artifactsStore, new IncludeProjectPropertiesConfiguration(properties, manifestProperties), manageDependenciesService, 3);


@Before
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@
import com.google.inject.Provides;
import com.google.inject.Singleton;
import org.finos.legend.depot.schedules.services.SchedulesFactory;
import org.finos.legend.depot.services.api.dependencies.ManageDependenciesService;
import org.finos.legend.depot.services.api.entities.ManageEntitiesService;
import org.finos.legend.depot.services.api.generation.file.ManageFileGenerationsService;
import org.finos.legend.depot.services.api.projects.ManageProjectsService;
import org.finos.legend.depot.services.api.versionedEntities.ManageVersionedEntitiesService;
import org.finos.legend.depot.services.dependencies.ManageDependenciesServiceImpl;
import org.finos.legend.depot.services.entities.ManageEntitiesServiceImpl;
import org.finos.legend.depot.services.generation.file.ManageFileGenerationsServiceImpl;
import org.finos.legend.depot.services.projects.ManageProjectsServiceImpl;
Expand Down Expand Up @@ -51,10 +53,12 @@ protected void configure()
bind(ManageEntitiesService.class).to(ManageEntitiesServiceImpl.class);
bind(ManageVersionedEntitiesService.class).to(ManageVersionedEntitiesServiceImpl.class);
bind(ManageFileGenerationsService.class).to(ManageFileGenerationsServiceImpl.class);
bind(ManageDependenciesService.class).to(ManageDependenciesServiceImpl.class);
bind(VersionsMismatchService.class);

expose(ManageProjectsService.class);
expose(ManageEntitiesService.class);
expose(ManageDependenciesService.class);
expose(ManageVersionedEntitiesService.class);
expose(ManageFileGenerationsService.class);
expose(VersionsMismatchService.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,24 @@
package org.finos.legend.depot.services;

import com.google.inject.PrivateModule;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.name.Names;
import org.finos.legend.depot.services.api.entities.EntitiesService;
import org.finos.legend.depot.services.api.entities.EntityClassifierService;
import org.finos.legend.depot.services.api.generation.file.FileGenerationsService;
import org.finos.legend.depot.services.api.projects.ProjectsService;
import org.finos.legend.depot.services.api.serverInfo.InfoService;
import org.finos.legend.depot.services.api.versionedEntities.VersionedEntitiesService;
import org.finos.legend.depot.services.dependencies.DependencyUtil;
import org.finos.legend.depot.services.entities.EntitiesServiceImpl;
import org.finos.legend.depot.services.entities.EntityClassifierServiceImpl;
import org.finos.legend.depot.services.generation.file.FileGenerationsServiceImpl;
import org.finos.legend.depot.services.projects.ProjectsServiceImpl;
import org.finos.legend.depot.services.versionedEntities.VersionedEntitiesServiceImpl;

import javax.inject.Named;

public class ReadDataServicesModule extends PrivateModule
{
@Override
Expand All @@ -47,5 +53,14 @@ protected void configure()
expose(EntityClassifierService.class);
expose(FileGenerationsService.class);
expose(InfoService.class);
expose(DependencyUtil.class).annotatedWith(Names.named("dependencyUtil"));
}

@Provides
@Named("dependencyUtil")
@Singleton
public DependencyUtil initialiseDependencyCache()
{
return new DependencyUtil();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright 2021 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package org.finos.legend.depot.services.api.dependencies;

import org.finos.legend.depot.domain.project.ProjectVersion;
import org.finos.legend.depot.domain.project.StoreProjectVersionData;
import org.finos.legend.depot.domain.project.dependencies.VersionDependencyReport;

import java.util.List;

public interface ManageDependenciesService
{
List<ProjectVersion> calculateDependencies(String groupId, String artifactId, String versionId);

VersionDependencyReport calculateTransitiveDependencies(List<ProjectVersion> directDependencies);

List<String> validateDependencies(List<ProjectVersion> dependencies, String versionId);

StoreProjectVersionData updateTransitiveDependencies(String groupId, String artifactId, String versionId);

void setProjectDataTransitiveDependencies(StoreProjectVersionData projectData);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2021 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package org.finos.legend.depot.services.dependencies;

import org.eclipse.collections.api.block.function.Function2;
import org.finos.legend.depot.domain.project.ProjectVersion;

import java.util.Set;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Collection;
import java.util.stream.Collectors;

public class DependencyUtil
{
public List<ProjectVersion> overrideDependencies(List<ProjectVersion> directDependencies, List<ProjectVersion> projectDependencies, Function2<List<ProjectVersion>, Boolean, Set<ProjectVersion>> executableFunction)
{
Map<String, List<ProjectVersion>> dependenciesLocator = projectDependencies.stream().collect(Collectors.groupingBy(dep -> dep.getGroupId() + dep.getArtifactId()));
Set<ProjectVersion> overriddenDependencies = directDependencies.stream().map(dep -> dependenciesLocator.getOrDefault(dep.getGroupId() + dep.getArtifactId(), Collections.emptyList())).flatMap(Collection::stream).collect(Collectors.toSet());
overriddenDependencies.removeAll(directDependencies);
Set<ProjectVersion> deleteDependencies = overriddenDependencies.parallelStream().map(dep -> executableFunction.apply(Collections.singletonList(dep), true)).flatMap(Collection::stream).collect(Collectors.toSet());
projectDependencies.removeAll(deleteDependencies);
projectDependencies.removeAll(overriddenDependencies);
return projectDependencies;
}
}
Loading

0 comments on commit 98402f0

Please sign in to comment.