diff --git a/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendCacheService.java b/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendCacheService.java index 1489a047..adde8baf 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendCacheService.java +++ b/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendCacheService.java @@ -17,47 +17,41 @@ * @since 23/05/2023 */ @Service -public class FrontendCacheService -{ - private record PluginIdAndVersion(String id, String version) - { - } - - private final RepositoryChannelsService myRepositoryChannelsService; - private final PluginStatisticsService myPluginStatisticsService; - - @Autowired - public FrontendCacheService(RepositoryChannelsService repositoryChannelsService, PluginStatisticsService pluginStatisticsService) - { - myRepositoryChannelsService = repositoryChannelsService; - myPluginStatisticsService = pluginStatisticsService; - } - - public Collection listPlugins() - { - Map map = new HashMap<>(); - for(PluginChannel channel : PluginChannel.values()) - { - RepositoryChannelStore service = myRepositoryChannelsService.getRepositoryByChannel(channel); - - service.iteratePluginNodes(pluginNode -> - { - FrontPluginNode node = map.computeIfAbsent(new PluginIdAndVersion(pluginNode.id, pluginNode.version), pluginIdAndVersion -> - { - FrontPluginNode frontPluginNode = new FrontPluginNode(); - frontPluginNode.myPluginNode = pluginNode.clone(); - - int countAll = myPluginStatisticsService.getDownloadStatCountAll(pluginNode.id); - - frontPluginNode.myPluginNode.downloads = countAll; - frontPluginNode.myPluginNode.downloadsAll = countAll; - return frontPluginNode; - }); - - node.myChannels.add(channel); - }); - } - - return map.values(); - } +public class FrontendCacheService { + private record PluginIdAndVersion(String id, String version) { + } + + private final RepositoryChannelsService myRepositoryChannelsService; + private final PluginStatisticsService myPluginStatisticsService; + + @Autowired + public FrontendCacheService(RepositoryChannelsService repositoryChannelsService, PluginStatisticsService pluginStatisticsService) { + myRepositoryChannelsService = repositoryChannelsService; + myPluginStatisticsService = pluginStatisticsService; + } + + public Collection listPlugins() { + Map map = new HashMap<>(); + for (PluginChannel channel : PluginChannel.values()) { + RepositoryChannelStore service = myRepositoryChannelsService.getRepositoryByChannel(channel); + + service.iteratePluginNodes(pluginNode -> { + FrontPluginNode node = map.computeIfAbsent(new PluginIdAndVersion(pluginNode.id, pluginNode.version), pluginIdAndVersion -> + { + FrontPluginNode frontPluginNode = new FrontPluginNode(); + frontPluginNode.myPluginNode = pluginNode.clone(); + + int countAll = myPluginStatisticsService.getDownloadStatCountAll(pluginNode.id); + + frontPluginNode.myPluginNode.downloads = countAll; + frontPluginNode.myPluginNode.downloadsAll = countAll; + return frontPluginNode; + }); + + node.myChannels.add(channel); + }); + } + + return map.values(); + } } diff --git a/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendRepositoryRestController.java b/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendRepositoryRestController.java index 54a9da39..507ef0dc 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendRepositoryRestController.java +++ b/hub-backend/src/main/java/consulo/hub/backend/frontend/FrontendRepositoryRestController.java @@ -2,6 +2,7 @@ import consulo.hub.backend.repository.PluginStatisticsService; import consulo.hub.backend.repository.RepositoryChannelIterationService; +import consulo.hub.backend.repository.cleanup.RepositoryCleanupService; import consulo.hub.shared.repository.FrontPluginNode; import consulo.hub.shared.repository.PluginChannel; import consulo.hub.shared.repository.domain.RepositoryDownloadInfo; @@ -18,37 +19,41 @@ * @since 22/08/2021 */ @RestController -public class FrontendRepositoryRestController -{ - private final PluginStatisticsService myPluginStatisticsService; - - private final RepositoryChannelIterationService myPluginChannelIterationService; - - private final FrontendCacheService myFrontendCacheService; - - public FrontendRepositoryRestController(PluginStatisticsService pluginStatisticsService, RepositoryChannelIterationService pluginChannelIterationService, FrontendCacheService frontendCacheService) - { - myPluginStatisticsService = pluginStatisticsService; - myPluginChannelIterationService = pluginChannelIterationService; - myFrontendCacheService = frontendCacheService; - } - - @RequestMapping("/api/private/repository/list") - public Collection listPlugins() - { - return myFrontendCacheService.listPlugins(); - } - - @RequestMapping("/api/private/repository/downloadStat") - public List downloadStat(@RequestParam("pluginId") String pluginId) - { - return myPluginStatisticsService.getDownloadStat(pluginId); - } - - @RequestMapping("/api/private/repository/iterate") - public Map iteratePlugins(@RequestParam("from") PluginChannel from, @RequestParam("to") PluginChannel to) - { - myPluginChannelIterationService.iterate(from, to); - return Map.of(); - } +public class FrontendRepositoryRestController { + private final PluginStatisticsService myPluginStatisticsService; + private final RepositoryChannelIterationService myPluginChannelIterationService; + private final FrontendCacheService myFrontendCacheService; + private final RepositoryCleanupService myRepositoryCleanupService; + + public FrontendRepositoryRestController(PluginStatisticsService pluginStatisticsService, + RepositoryChannelIterationService pluginChannelIterationService, + FrontendCacheService frontendCacheService, + RepositoryCleanupService repositoryCleanupService) { + myPluginStatisticsService = pluginStatisticsService; + myPluginChannelIterationService = pluginChannelIterationService; + myFrontendCacheService = frontendCacheService; + myRepositoryCleanupService = repositoryCleanupService; + } + + @RequestMapping("/api/private/repository/list") + public Collection listPlugins() { + return myFrontendCacheService.listPlugins(); + } + + @RequestMapping("/api/private/repository/downloadStat") + public List downloadStat(@RequestParam("pluginId") String pluginId) { + return myPluginStatisticsService.getDownloadStat(pluginId); + } + + @RequestMapping("/api/private/repository/iterate") + public Map iteratePlugins(@RequestParam("from") PluginChannel from, @RequestParam("to") PluginChannel to) { + myPluginChannelIterationService.iterate(from, to); + return Map.of(); + } + + @RequestMapping("/api/private/repository/cleanup") + public Map cleanup() { + myRepositoryCleanupService.runCleanUpAsync(); + return Map.of(); + } } diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/RepositoryNodeState.java b/hub-backend/src/main/java/consulo/hub/backend/repository/RepositoryNodeState.java index 728526ef..33def208 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/repository/RepositoryNodeState.java +++ b/hub-backend/src/main/java/consulo/hub/backend/repository/RepositoryNodeState.java @@ -4,6 +4,7 @@ import consulo.hub.shared.repository.PluginNode; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import org.springframework.lang.NonNull; import java.util.List; import java.util.NavigableMap; @@ -14,27 +15,25 @@ * @author VISTALL * @since 18/05/2023 */ -public interface RepositoryNodeState -{ - @Nullable - PluginNode select(@Nonnull String platformVersion, @Nullable String version, boolean platformBuildSelect); +public interface RepositoryNodeState { + @Nullable + PluginNode select(@Nonnull String platformVersion, @Nullable String version, boolean platformBuildSelect); - default void runOver(@Nonnull String platformVersion, @Nullable String version, boolean platformBuildSelect, Consumer consumer) - { - PluginNode select = select(platformVersion, version, platformBuildSelect); - if(select != null) - { - consumer.accept(select); - } - } + default void runOver(@Nonnull String platformVersion, @Nullable String version, boolean platformBuildSelect, Consumer consumer) { + PluginNode select = select(platformVersion, version, platformBuildSelect); + if (select != null) { + consumer.accept(select); + } + } - void selectInto(@Nonnull PluginStatisticsService statisticsService, @Nonnull PluginChannel channel, @Nonnull String platformVersion, boolean platformBuildSelect, List list); + void selectInto(@Nonnull PluginStatisticsService statisticsService, @Nonnull PluginChannel channel, @Nonnull String platformVersion, boolean platformBuildSelect, List list); - boolean isInRepository(String version, String platformVersion); + boolean isInRepository(String version, String platformVersion); - void forEach(@Nonnull Consumer consumer); + void forEach(@Nonnull Consumer consumer); - void remove(String version, String platformVersion); + void remove(String version, String platformVersion); + @NonNull NavigableMap> getPluginsByPlatformVersion(); } diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/cleanup/RepositoryCleanupService.java b/hub-backend/src/main/java/consulo/hub/backend/repository/cleanup/RepositoryCleanupService.java index 41d82173..916b56f2 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/repository/cleanup/RepositoryCleanupService.java +++ b/hub-backend/src/main/java/consulo/hub/backend/repository/cleanup/RepositoryCleanupService.java @@ -9,8 +9,12 @@ import consulo.hub.shared.repository.util.PlatformNodeDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.task.TaskExecutor; import org.springframework.stereotype.Service; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.*; /** @@ -19,17 +23,25 @@ */ @Service public class RepositoryCleanupService { - public static final int ourMaxBuildCount = 10; + public static final int ourMaxBuildCount = 25; + + private static final int ourMaxRemovePerSession = 100; private static final Logger logger = LoggerFactory.getLogger(RepositoryCleanupService.class); private final RepositoryChannelsService myRepositoryChannelsService; + private final TaskExecutor myTaskExecutor; - public RepositoryCleanupService(RepositoryChannelsService repositoryChannelsService) { + public RepositoryCleanupService(RepositoryChannelsService repositoryChannelsService, TaskExecutor taskExecutor) { myRepositoryChannelsService = repositoryChannelsService; + myTaskExecutor = taskExecutor; + } + + public void runCleanUpAsync() { + myTaskExecutor.execute(this::runCleanUp); } - public void collect() { + public void runCleanUp() { Map>> versions = new LinkedHashMap<>(); PluginChannel[] pluginChannels = PluginChannel.values(); @@ -59,7 +71,7 @@ public void collect() { } } - Set allVersions = new TreeSet<>(); + TreeSet allVersions = new TreeSet<>(); // collect all data for (PlatformNodeDesc desc : PlatformNodeDesc.values()) { @@ -67,7 +79,8 @@ public void collect() { if (map.size() != pluginChannels.length) { versions.remove(desc); - } else { + } + else { for (TreeSet set : map.values()) { allVersions.addAll(set); } @@ -78,7 +91,6 @@ public void collect() { Map> map = versions.getOrDefault(desc, Map.of()); for (Map.Entry> entry : map.entrySet()) { - PluginChannel channel = entry.getKey(); TreeSet deployedVersions = entry.getValue(); Iterator allVersionsIterator = allVersions.iterator(); @@ -87,38 +99,102 @@ public void collect() { if (!deployedVersions.contains(allVersion)) { allVersionsIterator.remove(); + } + } + } + } + + List toRemoveBuilds = new ArrayList<>(ourMaxRemovePerSession); + while (allVersions.size() > ourMaxBuildCount) { + BuildNumber toRemove = allVersions.removeFirst(); + + toRemoveBuilds.add(toRemove.asString()); + + if (toRemoveBuilds.size() == ourMaxRemovePerSession) { + break; + } + } + + Set filesToRemove = new LinkedHashSet<>(); + + for (PlatformNodeDesc desc : PlatformNodeDesc.values()) { + for (PluginChannel channel : PluginChannel.values()) { + RepositoryChannelStore store = myRepositoryChannelsService.getRepositoryByChannel(channel); + + RepositoryNodeState state = store.getState(desc.id()); + if (state == null) { + continue; + } + + for (String toRemoveBuild : toRemoveBuilds) { + PluginNode node = state.select(toRemoveBuild, toRemoveBuild, true); + if (node == null) { + continue; + } + + filesToRemove.add(Objects.requireNonNull(node.targetPath)); - System.out.println("skip " + allVersion + " not in " + channel + " channed"); + state.remove(toRemoveBuild, toRemoveBuild); + } + } + } + + Set allPluginIds = new HashSet<>(); + + for (PluginChannel channel : PluginChannel.values()) { + RepositoryChannelStore store = myRepositoryChannelsService.getRepositoryByChannel(channel); + + store.iteratePluginNodes(pluginNode -> allPluginIds.add(pluginNode.id)); + } + + for (PlatformNodeDesc desc : PlatformNodeDesc.values()) { + allPluginIds.remove(desc.id()); + for (String oldId : desc.oldIds()) { + allPluginIds.remove(oldId); + } + } + + for (PluginChannel channel : PluginChannel.values()) { + RepositoryChannelStore store = myRepositoryChannelsService.getRepositoryByChannel(channel); + + for (String allPluginId : allPluginIds) { + RepositoryNodeState state = store.getState(allPluginId); + if (state == null) { + continue; + } + + for (String toRemoveBuild : toRemoveBuilds) { + NavigableSet pluginNodes = state.getPluginsByPlatformVersion().get(toRemoveBuild); + if (pluginNodes == null) { + continue; + } + + for (PluginNode node : pluginNodes) { + filesToRemove.add(Objects.requireNonNull(node.targetPath)); + + state.remove(node.version, node.platformVersion); } } } } + logger.info("CleanUp: Analyzing repos - marked {} platform builds to remove. All {} files marked to remove with plugins", toRemoveBuilds.size(), filesToRemove.size()); + + long startTime = System.currentTimeMillis(); + filesToRemove.parallelStream().forEach(path -> { + try { + Files.deleteIfExists(path); + + Path jsonFile = path.getParent().resolve(path.getFileName() + ".json"); + + Files.deleteIfExists(jsonFile); + } catch (IOException e) { + logger.error("Failed to remove " + path, e); + } + }); + + long endTime = (System.currentTimeMillis() - startTime) / 1000L; -// Map pluginStates = pluginChannelService.copyPluginsState(); -// // first of all we need check platform nodes -// for (String platformPluginId : RepositoryUtil.ourPlatformPluginIds) { -// PluginsStateOld pluginsState = pluginStates.get(platformPluginId); -// if (pluginsState == null) { -// continue; -// } -// -// NavigableMap> map = pluginsState.getPluginsByPlatformVersion(); -// -// int i = map.size(); -// for (Map.Entry> entry : map.entrySet()) { -// String platformVersion = entry.getKey(); -// -// if (i > ourMaxBuildCount) { -// outdatedPlatformVersions.add(platformVersion); -// NavigableSet value = entry.getValue(); -// if (!value.isEmpty()) { -// toRemove.add(value.iterator().next()); -// } -// -// i--; -// } -// } -// } + logger.info("CleanUp: Finished to remove files in {} seconds", endTime); } } diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/cron/PluginsCleanuper.java b/hub-backend/src/main/java/consulo/hub/backend/repository/cron/PluginsCleanuper.java deleted file mode 100644 index 78a21c21..00000000 --- a/hub-backend/src/main/java/consulo/hub/backend/repository/cron/PluginsCleanuper.java +++ /dev/null @@ -1,28 +0,0 @@ -package consulo.hub.backend.repository.cron; - -import consulo.hub.backend.repository.RepositoryChannelIterationService; -import consulo.hub.backend.repository.RepositoryChannelsService; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -/** - * @author VISTALL - * @since 2025-02-09 - */ -@Service -public class PluginsCleanuper { - - private final RepositoryChannelsService myRepositoryChannelsService; - private final RepositoryChannelIterationService myRepositoryChannelIterationService; - - public PluginsCleanuper(RepositoryChannelsService repositoryChannelsService, - RepositoryChannelIterationService repositoryChannelIterationService) { - myRepositoryChannelsService = repositoryChannelsService; - myRepositoryChannelIterationService = repositoryChannelIterationService; - } - - @Scheduled(cron = "0 * * * * *") - public void cleanup() { - // TODO - } -} diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/cron/RepositoryCleanuper.java b/hub-backend/src/main/java/consulo/hub/backend/repository/cron/RepositoryCleanuper.java new file mode 100644 index 00000000..60af6737 --- /dev/null +++ b/hub-backend/src/main/java/consulo/hub/backend/repository/cron/RepositoryCleanuper.java @@ -0,0 +1,26 @@ +package consulo.hub.backend.repository.cron; + +import consulo.hub.backend.repository.cleanup.RepositoryCleanupService; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * @author VISTALL + * @since 2025-02-09 + */ +@Service +public class RepositoryCleanuper { + + private final RepositoryCleanupService myRepositoryChannelsService; + + public RepositoryCleanuper(RepositoryCleanupService repositoryChannelsService) { + myRepositoryChannelsService = repositoryChannelsService; + } + + @Scheduled(cron = "0 * * * * *") + public void cleanup() { + if (Boolean.FALSE) { + myRepositoryChannelsService.runCleanUpAsync(); + } + } +} diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/BaseRepositoryNodeState.java b/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/BaseRepositoryNodeState.java index 1c75cfe6..2d9c73a1 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/BaseRepositoryNodeState.java +++ b/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/BaseRepositoryNodeState.java @@ -45,7 +45,13 @@ protected BaseRepositoryNodeState(String pluginId) { @Nonnull public NavigableMap> getPluginsByPlatformVersion() { try (AccessToken ignored = readLock()) { - return new TreeMap<>(myPluginsByPlatformVersion); + TreeMap> copy = new TreeMap<>(); + for (Map.Entry> entry : myPluginsByPlatformVersion.entrySet()) { + NavigableSet copySet = newTreeSet(""); + copySet.addAll(entry.getValue()); + copy.put(entry.getKey(), copySet); + } + return copy; } } diff --git a/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/neww/NewRepositoryNodeState.java b/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/neww/NewRepositoryNodeState.java index 034a4944..f88cb8f2 100644 --- a/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/neww/NewRepositoryNodeState.java +++ b/hub-backend/src/main/java/consulo/hub/backend/repository/impl/store/neww/NewRepositoryNodeState.java @@ -13,46 +13,40 @@ * @author VISTALL * @since 18/05/2023 */ -public class NewRepositoryNodeState extends BaseRepositoryNodeState -{ - private final NewInlineRepositoryStore myInlineRepositoryStore; - private final PluginChannel myPluginChannel; - - public NewRepositoryNodeState(PluginChannel channel, String pluginId, NewInlineRepositoryStore inlineRepositoryStore) - { - super(pluginId); - myPluginChannel = channel; - myInlineRepositoryStore = inlineRepositoryStore; - } - - @Override - public void push(PluginNode pluginNode, String ext, ThrowableConsumer writeConsumer) throws Exception - { - try (AccessToken ignored = writeLock()) - { - Path artifactPath = myInlineRepositoryStore.prepareArtifactPath(pluginNode.id, pluginNode.version, ext); - - writeConsumer.consume(artifactPath); - - prepareNode(pluginNode, artifactPath); - - pluginNode.length = Files.size(artifactPath); - pluginNode.targetPath = artifactPath; - pluginNode.cleanUp(); - - _add(pluginNode); - - myInlineRepositoryStore.updateMeta(pluginNode.id, pluginNode.version, ext, meta -> - { - meta.node = pluginNode; - meta.channels.add(myPluginChannel); - }); - } - } - - @Override - protected void removeRepositoryArtifact(PluginNode target) - { - throw new UnsupportedOperationException(); - } +public class NewRepositoryNodeState extends BaseRepositoryNodeState { + private final NewInlineRepositoryStore myInlineRepositoryStore; + private final PluginChannel myPluginChannel; + + public NewRepositoryNodeState(PluginChannel channel, String pluginId, NewInlineRepositoryStore inlineRepositoryStore) { + super(pluginId); + myPluginChannel = channel; + myInlineRepositoryStore = inlineRepositoryStore; + } + + @Override + public void push(PluginNode pluginNode, String ext, ThrowableConsumer writeConsumer) throws Exception { + try (AccessToken ignored = writeLock()) { + Path artifactPath = myInlineRepositoryStore.prepareArtifactPath(pluginNode.id, pluginNode.version, ext); + + writeConsumer.consume(artifactPath); + + prepareNode(pluginNode, artifactPath); + + pluginNode.length = Files.size(artifactPath); + pluginNode.targetPath = artifactPath; + pluginNode.cleanUp(); + + _add(pluginNode); + + myInlineRepositoryStore.updateMeta(pluginNode.id, pluginNode.version, ext, meta -> + { + meta.node = pluginNode; + meta.channels.add(myPluginChannel); + }); + } + } + + @Override + protected void removeRepositoryArtifact(PluginNode target) { + } } diff --git a/hub-backend/src/test/java/consulo/webservice/RepositoryCleanupTest.java b/hub-backend/src/test/java/consulo/webservice/RepositoryCleanupTest.java index 89e026a8..0889084a 100644 --- a/hub-backend/src/test/java/consulo/webservice/RepositoryCleanupTest.java +++ b/hub-backend/src/test/java/consulo/webservice/RepositoryCleanupTest.java @@ -46,7 +46,7 @@ public static void before() throws Exception { ourPluginChannelsService.init(); - ourRepositoryCleanupService = new RepositoryCleanupService(ourPluginChannelsService); + ourRepositoryCleanupService = new RepositoryCleanupService(ourPluginChannelsService, Runnable::run); int maxVersion = 10000; List versions = IntStream.range(1, maxVersion).mapToObj(Integer::valueOf).toList(); @@ -79,6 +79,7 @@ public static void before() throws Exception { pluginNode.id = PLATFORM_ID; pluginNode.version = String.valueOf(deployVersion); pluginNode.platformVersion = String.valueOf(deployVersion); + pluginNode.targetPath = Path.of(PLATFORM_ID + "_" + deployVersion + ".platform"); store._add(pluginNode); } @@ -92,6 +93,6 @@ public static void after() throws Exception { @Test public void test() { - ourRepositoryCleanupService.collect(); + ourRepositoryCleanupService.runCleanUp(); } } diff --git a/hub-frontend/src/main/java/consulo/hub/frontend/vflow/backend/service/BackendRepositoryService.java b/hub-frontend/src/main/java/consulo/hub/frontend/vflow/backend/service/BackendRepositoryService.java index 38324bd7..c6efc293 100644 --- a/hub-frontend/src/main/java/consulo/hub/frontend/vflow/backend/service/BackendRepositoryService.java +++ b/hub-frontend/src/main/java/consulo/hub/frontend/vflow/backend/service/BackendRepositoryService.java @@ -20,53 +20,54 @@ * @since 21/08/2021 */ @Service -public class BackendRepositoryService -{ - private static final Logger LOG = LoggerFactory.getLogger(BackendRepositoryService.class); +public class BackendRepositoryService { + private static final Logger LOG = LoggerFactory.getLogger(BackendRepositoryService.class); - @Autowired - private ApiBackendRequestor myApiBackendRequestor; + @Autowired + private ApiBackendRequestor myApiBackendRequestor; - public void listAll(@Nonnull Consumer consumer) - { - try - { - FrontPluginNode[] nodes = myApiBackendRequestor.runRequest(BackendApiUrl.toPrivate("/repository/list"), Map.of(), FrontPluginNode[].class); - if(nodes == null) - { - nodes = new FrontPluginNode[0]; - } + public void cleanup() { + try { + myApiBackendRequestor.runRequest(BackendApiUrl.toPrivate("/repository/cleanup"), Map.of(), new TypeReference>() { + }); + } + catch (BackendServiceDownException e) { + throw e; + } + catch (Exception e) { + LOG.error("Fail to get plugins", e); + } + } - for(FrontPluginNode node : nodes) - { - consumer.accept(node); - } - } - catch(BackendServiceDownException e) - { - throw e; - } - catch(Exception e) - { - LOG.error("Fail to get plugins", e); - } - } + public void listAll(@Nonnull Consumer consumer) { + try { + FrontPluginNode[] nodes = myApiBackendRequestor.runRequest(BackendApiUrl.toPrivate("/repository/list"), Map.of(), FrontPluginNode[].class); + if (nodes == null) { + nodes = new FrontPluginNode[0]; + } - public void iteratePlugins(@Nonnull PluginChannel from, @Nonnull PluginChannel to) - { - try - { - myApiBackendRequestor.runRequest(BackendApiUrl.toPrivate("/repository/iterate"), Map.of("from", from.name(), "to", to.name()), new TypeReference>() - { - }); - } - catch(BackendServiceDownException e) - { - throw e; - } - catch(Exception e) - { - LOG.error("Fail iterate plugins, from=" + from + " to=" + to, e); - } - } + for (FrontPluginNode node : nodes) { + consumer.accept(node); + } + } + catch (BackendServiceDownException e) { + throw e; + } + catch (Exception e) { + LOG.error("Fail to get plugins", e); + } + } + + public void iteratePlugins(@Nonnull PluginChannel from, @Nonnull PluginChannel to) { + try { + myApiBackendRequestor.runRequest(BackendApiUrl.toPrivate("/repository/iterate"), Map.of("from", from.name(), "to", to.name()), new TypeReference>() { + }); + } + catch (BackendServiceDownException e) { + throw e; + } + catch (Exception e) { + LOG.error("Fail iterate plugins, from=" + from + " to=" + to, e); + } + } } diff --git a/hub-frontend/src/main/java/consulo/hub/frontend/vflow/repository/view/AdminRepositoryView.java b/hub-frontend/src/main/java/consulo/hub/frontend/vflow/repository/view/AdminRepositoryView.java index 2070ee81..0a65355c 100644 --- a/hub-frontend/src/main/java/consulo/hub/frontend/vflow/repository/view/AdminRepositoryView.java +++ b/hub-frontend/src/main/java/consulo/hub/frontend/vflow/repository/view/AdminRepositoryView.java @@ -1,22 +1,20 @@ package consulo.hub.frontend.vflow.repository.view; -import com.vaadin.flow.component.ClickEvent; -import com.vaadin.flow.component.ComponentEventListener; import com.vaadin.flow.component.button.Button; import com.vaadin.flow.component.orderedlayout.HorizontalLayout; +import com.vaadin.flow.component.orderedlayout.VerticalLayout; import com.vaadin.flow.router.PageTitle; import com.vaadin.flow.router.Route; import consulo.hub.frontend.vflow.backend.service.BackendRepositoryService; import consulo.hub.frontend.vflow.base.MainLayout; -import consulo.procoeton.core.vaadin.ui.VChildLayout; -import consulo.procoeton.core.vaadin.ui.util.VaadinUIUtil; import consulo.hub.shared.auth.Roles; import consulo.hub.shared.repository.PluginChannel; +import consulo.procoeton.core.vaadin.ui.VChildLayout; +import consulo.procoeton.core.vaadin.ui.util.VaadinUIUtil; +import jakarta.annotation.Nonnull; import jakarta.annotation.security.RolesAllowed; import org.springframework.beans.factory.annotation.Autowired; -import jakarta.annotation.Nonnull; - /** * @author VISTALL * @since 05-Jan-17 @@ -24,40 +22,37 @@ @PageTitle("Admin/Repository") @Route(value = "admin/repository", layout = MainLayout.class) @RolesAllowed(Roles.ROLE_SUPERUSER) -public class AdminRepositoryView extends VChildLayout -{ - private BackendRepositoryService myBackendRepositoryService; - - @Autowired - public AdminRepositoryView(BackendRepositoryService backendRepositoryService) - { - myBackendRepositoryService = backendRepositoryService; - - setMargin(false); - setSpacing(false); - setSizeFull(); - //setDefaultComponentAlignment(Alignment.TOP_LEFT); - - HorizontalLayout layout = VaadinUIUtil.newHorizontalLayout(); - layout.setSpacing(true); - ComponentEventListener> listener2 = event -> forceIterate(PluginChannel.nightly, PluginChannel.alpha); - layout.add(new Button("nightly " + rightArrow() + " alpha", listener2)); - ComponentEventListener> listener1 = event -> forceIterate(PluginChannel.alpha, PluginChannel.beta); - layout.add(new Button("alpha " + rightArrow() + " beta", listener1)); - ComponentEventListener> listener = event -> forceIterate(PluginChannel.beta, PluginChannel.release); - layout.add(new Button("beta " + rightArrow() + " release", listener)); - - add(layout); - } - - @Nonnull - private static String rightArrow() - { - return "->"; - } - - private void forceIterate(@Nonnull PluginChannel from, @Nonnull PluginChannel to) - { - myBackendRepositoryService.iteratePlugins(from, to); - } +public class AdminRepositoryView extends VChildLayout { + private BackendRepositoryService myBackendRepositoryService; + + @Autowired + public AdminRepositoryView(BackendRepositoryService backendRepositoryService) { + myBackendRepositoryService = backendRepositoryService; + + setMargin(false); + setSpacing(false); + setSizeFull(); + + VerticalLayout rowLayout = VaadinUIUtil.newVerticalLayout(); + add(rowLayout); + + HorizontalLayout layout = VaadinUIUtil.newHorizontalLayout(); + layout.setSpacing(true); + rowLayout.add(layout); + + layout.add(new Button("nightly " + rightArrow() + " alpha", e -> forceIterate(PluginChannel.nightly, PluginChannel.alpha))); + layout.add(new Button("alpha " + rightArrow() + " beta", e -> forceIterate(PluginChannel.alpha, PluginChannel.beta))); + layout.add(new Button("beta " + rightArrow() + " release", e -> forceIterate(PluginChannel.beta, PluginChannel.release))); + + rowLayout.add(new Button("Run Cleanup", e -> myBackendRepositoryService.cleanup())); + } + + @Nonnull + private static String rightArrow() { + return "->"; + } + + private void forceIterate(@Nonnull PluginChannel from, @Nonnull PluginChannel to) { + myBackendRepositoryService.iteratePlugins(from, to); + } } diff --git a/procoeton-ui-core/src/main/java/consulo/procoeton/core/backend/ApiBackendRequestor.java b/procoeton-ui-core/src/main/java/consulo/procoeton/core/backend/ApiBackendRequestor.java index 2833a2be..d7b40cea 100644 --- a/procoeton-ui-core/src/main/java/consulo/procoeton/core/backend/ApiBackendRequestor.java +++ b/procoeton-ui-core/src/main/java/consulo/procoeton/core/backend/ApiBackendRequestor.java @@ -33,128 +33,106 @@ * @since 22/08/2021 */ @Service -public class ApiBackendRequestor -{ - private final ObjectMapper myObjectMapper; - - private final ProPropertiesService myPropertiesService; - - private final LogoutService myLogoutService; - - private CloseableHttpClient myClient = HttpClients.custom().setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(5000).setSocketTimeout(5000).build()).build(); - - private final String myHostKey = ApiBackendKeys.BACKEND_HOST_URL_KEY; - - @Autowired - public ApiBackendRequestor(ObjectMapper objectMapper, ProPropertiesService propertiesService, LogoutService logoutService) - { - myObjectMapper = objectMapper; - myPropertiesService = propertiesService; - myLogoutService = logoutService; - } - - @Nullable - public T runRequest(BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz) throws Exception - { - return runRequest(urlSuffix, parameters, valueClazz, () -> null); - } - - @Nullable - public T runRequest(BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz, Supplier defaultValueGetter) throws Exception - { - String host = "http://localhost:2233"; - - if(myPropertiesService.isInstalled()) - { - PropertySet propertySet = myPropertiesService.getPropertySet(); - host = propertySet.getStringProperty(myHostKey); - } - - return runRequest(host, urlSuffix, parameters, valueClazz, defaultValueGetter); - } - - public T runRequest(BackendApiUrl urlSuffix, Map parameters, Class valueClazz) throws Exception - { - return runRequest(urlSuffix, parameters, valueClazz, () -> null); - } - - public T runRequest(BackendApiUrl urlSuffix, Map parameters, Class valueClazz, Supplier defaultValueGetter) throws Exception - { - return runRequest(urlSuffix, parameters, new TypeReference() - { - @Override - public Type getType() - { - return valueClazz; - } - }, defaultValueGetter); - } - - public T runRequest(String host, BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz) throws Exception - { - return runRequest(host, urlSuffix, parameters, valueClazz, () -> null); - } - - public T runRequest(String host, BackendApiUrl url, Map parameters, TypeReference valueClazz, Supplier defaultValueGetter) throws Exception - { - RequestBuilder builder = RequestBuilder.get(url.build(host)); - builder.addHeader("Content-Type", "application/json"); - - for(Map.Entry entry : parameters.entrySet()) - { - builder.addParameter(entry.getKey(), entry.getValue()); - } - - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - if(authentication instanceof BackendAuthenticationToken) - { - builder.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + ((BackendAuthenticationToken) authentication).getToken()); - } - - if(myPropertiesService.isInstalled() && url.isPrivate()) - { - builder.addHeader(ServicesHeaders.BACKEND_SECURE_KEY, myPropertiesService.getPropertySet().getStringProperty(ApiBackendKeys.BACKEND_SECURE_KEY, "")); - } - - try - { - T value = myClient.execute(builder.build(), response -> - { - int statusCode = response.getStatusLine().getStatusCode(); - if(statusCode == 401 || statusCode == 403) - { - myLogoutService.logout(UI.getCurrent(), false); - } - - if(statusCode != 200) - { - throw new IOException("request failed. Code: " + statusCode); - } - - String json = EntityUtils.toString(response.getEntity()); - return myObjectMapper.readValue(json, valueClazz); - }); - return value == null ? defaultValueGetter.get() : value; - } - catch(HttpHostConnectException | HttpConnectTimeoutException e) - { - throw new BackendServiceDownException(e); - } - catch(IOException ignored) - { - return defaultValueGetter.get(); - } - } - - @PreDestroy - public void destroy() - { - try - { - myClient.close(); - } - catch(IOException ignored) - { - } - } +public class ApiBackendRequestor { + private final ObjectMapper myObjectMapper; + + private final ProPropertiesService myPropertiesService; + + private final LogoutService myLogoutService; + + private CloseableHttpClient myClient = HttpClients.custom().setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(5000).setSocketTimeout(5000).build()).build(); + + private final String myHostKey = ApiBackendKeys.BACKEND_HOST_URL_KEY; + + @Autowired + public ApiBackendRequestor(ObjectMapper objectMapper, ProPropertiesService propertiesService, LogoutService logoutService) { + myObjectMapper = objectMapper; + myPropertiesService = propertiesService; + myLogoutService = logoutService; + } + + @Nullable + public T runRequest(BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz) throws Exception { + return runRequest(urlSuffix, parameters, valueClazz, () -> null); + } + + @Nullable + public T runRequest(BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz, Supplier defaultValueGetter) throws Exception { + String host = "http://localhost:2233"; + + if (myPropertiesService.isInstalled()) { + PropertySet propertySet = myPropertiesService.getPropertySet(); + host = propertySet.getStringProperty(myHostKey); + } + + return runRequest(host, urlSuffix, parameters, valueClazz, defaultValueGetter); + } + + public T runRequest(BackendApiUrl urlSuffix, Map parameters, Class valueClazz) throws Exception { + return runRequest(urlSuffix, parameters, valueClazz, () -> null); + } + + public T runRequest(BackendApiUrl urlSuffix, Map parameters, Class valueClazz, Supplier defaultValueGetter) throws Exception { + return runRequest(urlSuffix, parameters, new TypeReference() { + @Override + public Type getType() { + return valueClazz; + } + }, defaultValueGetter); + } + + public T runRequest(String host, BackendApiUrl urlSuffix, Map parameters, TypeReference valueClazz) throws Exception { + return runRequest(host, urlSuffix, parameters, valueClazz, () -> null); + } + + public T runRequest(String host, BackendApiUrl url, Map parameters, TypeReference valueClazz, Supplier defaultValueGetter) throws Exception { + RequestBuilder builder = RequestBuilder.get(url.build(host)); + builder.addHeader("Content-Type", "application/json"); + + for (Map.Entry entry : parameters.entrySet()) { + builder.addParameter(entry.getKey(), entry.getValue()); + } + + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + if (authentication instanceof BackendAuthenticationToken) { + builder.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + ((BackendAuthenticationToken) authentication).getToken()); + } + + if (myPropertiesService.isInstalled() && url.isPrivate()) { + builder.addHeader(ServicesHeaders.BACKEND_SECURE_KEY, myPropertiesService.getPropertySet().getStringProperty(ApiBackendKeys.BACKEND_SECURE_KEY, "")); + } + + try { + T value = myClient.execute(builder.build(), response -> + { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 401 || statusCode == 403) { + myLogoutService.logout(UI.getCurrent(), false); + } + + if (statusCode != 200) { + throw new IOException("request failed. Code: " + statusCode); + } + + String json = EntityUtils.toString(response.getEntity()); + return myObjectMapper.readValue(json, valueClazz); + }); + return value == null ? defaultValueGetter.get() : value; + } + catch (HttpHostConnectException | HttpConnectTimeoutException e) { + throw new BackendServiceDownException(e); + } + catch (IOException ignored) { + return defaultValueGetter.get(); + } + } + + @PreDestroy + public void destroy() { + try { + myClient.close(); + } + catch (IOException ignored) { + } + } }