From 238e161988b4250e0d55db80301d8a791a68ee0e Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 6 Apr 2023 19:48:26 +0300 Subject: [PATCH] Use AutoCloseable TempFile Added AutoCloseable TempFile, so that a created file is always deleted after use. --- .../eclipse/openvsx/ExtensionProcessor.java | 39 +++++----- .../org/eclipse/openvsx/ExtensionService.java | 12 +-- .../java/org/eclipse/openvsx/UserService.java | 10 +-- .../ExtractResourcesJobRequestHandler.java | 7 +- ...ExtractVsixManifestsJobRequestHandler.java | 9 +-- .../FixTargetPlatformsJobRequestHandler.java | 24 +++--- ...nerateSha256ChecksumJobRequestHandler.java | 6 +- .../openvsx/migration/MigrationService.java | 21 ++--- .../RenameDownloadsJobRequestHandler.java | 17 +++-- .../SetPreReleaseJobRequestHandler.java | 13 ++-- .../migration/SetPreReleaseJobService.java | 76 +------------------ .../mirror/MirrorExtensionService.java | 54 +++++-------- .../PublishExtensionVersionHandler.java | 9 +-- .../PublishExtensionVersionService.java | 17 ++--- .../storage/AzureBlobStorageService.java | 23 +++--- .../storage/AzureDownloadCountService.java | 27 +++---- .../storage/GoogleCloudStorageService.java | 24 ++---- .../openvsx/storage/IStorageService.java | 13 ++-- .../openvsx/storage/StorageUtilService.java | 28 +++---- .../org/eclipse/openvsx/util/TempFile.java | 32 ++++++++ .../openvsx/ExtensionProcessorTest.java | 49 ++++++------ 21 files changed, 204 insertions(+), 306 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/util/TempFile.java diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java index fca6cc1f0..f3c432d5e 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java @@ -9,37 +9,32 @@ ********************************************************************************/ package org.eclipse.openvsx; -import java.io.EOFException; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.zip.ZipException; -import java.util.zip.ZipFile; - import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.MissingNode; import com.fasterxml.jackson.dataformat.xml.XmlMapper; - import org.apache.commons.codec.digest.DigestUtils; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; -import org.eclipse.openvsx.util.ArchiveUtil; -import org.eclipse.openvsx.util.ErrorResultException; -import org.eclipse.openvsx.util.LicenseDetection; -import org.eclipse.openvsx.util.TargetPlatform; +import org.eclipse.openvsx.util.*; import org.elasticsearch.common.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.data.util.Pair; +import java.io.EOFException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + /** * Processes uploaded extension files and extracts their metadata. */ @@ -55,12 +50,12 @@ public class ExtensionProcessor implements AutoCloseable { protected final Logger logger = LoggerFactory.getLogger(ExtensionProcessor.class); - private final Path extensionFile; + private final TempFile extensionFile; private ZipFile zipFile; private JsonNode packageJson; private JsonNode vsixManifest; - public ExtensionProcessor(Path extensionFile) { + public ExtensionProcessor(TempFile extensionFile) { this.extensionFile = extensionFile; } @@ -80,7 +75,7 @@ private void readInputStream() { return; } try { - zipFile = new ZipFile(extensionFile.toFile()); + zipFile = new ZipFile(extensionFile.getPath().toFile()); } catch (ZipException exc) { throw new ErrorResultException("Could not read zip file: " + exc.getMessage()); } catch (EOFException exc) { @@ -357,7 +352,7 @@ public String getBinaryName(ExtensionVersion extVersion) { public FileResource generateSha256Checksum(ExtensionVersion extVersion) { String hash = null; - try(var input = Files.newInputStream(extensionFile)) { + try(var input = Files.newInputStream(extensionFile.getPath())) { hash = DigestUtils.sha256Hex(input); } catch (IOException e) { logger.error("Failed to read extensionFile", e); diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java index d70c3e43e..b3f74a494 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionService.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionService.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; -import java.nio.file.Path; import java.time.LocalDateTime; import java.util.LinkedHashSet; @@ -28,6 +27,7 @@ import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.util.ErrorResultException; +import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.TimeUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -55,7 +55,7 @@ public class ExtensionService { boolean requireLicense; @Transactional - public ExtensionVersion mirrorVersion(Path extensionFile, PersonalAccessToken token, String binaryName, String timestamp) { + public ExtensionVersion mirrorVersion(TempFile extensionFile, PersonalAccessToken token, String binaryName, String timestamp) { var download = doPublish(extensionFile, token, TimeUtil.fromUTCString(timestamp), false); publishHandler.mirror(download, extensionFile); download.setName(binaryName); @@ -69,7 +69,7 @@ public ExtensionVersion publishVersion(InputStream content, PersonalAccessToken return download.getExtension(); } - private FileResource doPublish(Path extensionFile, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) { + private FileResource doPublish(TempFile extensionFile, PersonalAccessToken token, LocalDateTime timestamp, boolean checkDependencies) { try (var processor = new ExtensionProcessor(extensionFile)) { var extVersion = publishHandler.createExtensionVersion(processor, token, timestamp, checkDependencies); if (requireLicense) { @@ -82,7 +82,7 @@ private FileResource doPublish(Path extensionFile, PersonalAccessToken token, Lo } } - private Path createExtensionFile(InputStream content) { + private TempFile createExtensionFile(InputStream content) { try (var input = new BufferedInputStream(content)) { input.mark(0); var skipped = input.skip(MAX_CONTENT_SIZE + 1); @@ -90,8 +90,8 @@ private Path createExtensionFile(InputStream content) { throw new ErrorResultException("The extension package exceeds the size limit of 512 MB.", HttpStatus.PAYLOAD_TOO_LARGE); } - var extensionFile = Files.createTempFile("extension_", ".vsix"); - try(var out = Files.newOutputStream(extensionFile)) { + var extensionFile = new TempFile("extension_", ".vsix"); + try(var out = Files.newOutputStream(extensionFile.getPath())) { input.reset(); input.transferTo(out); } diff --git a/server/src/main/java/org/eclipse/openvsx/UserService.java b/server/src/main/java/org/eclipse/openvsx/UserService.java index e23b38269..912ff8f74 100644 --- a/server/src/main/java/org/eclipse/openvsx/UserService.java +++ b/server/src/main/java/org/eclipse/openvsx/UserService.java @@ -241,10 +241,10 @@ public ResultJson updateNamespaceDetails(NamespaceDetailsJson details) { } boolean contentEquals; - var oldLogo = storageUtil.downloadNamespaceLogo(namespace); try ( + var oldLogo = storageUtil.downloadNamespaceLogo(namespace); var newLogoInput = new ByteArrayInputStream(details.logoBytes); - var oldLogoInput = Files.newInputStream(oldLogo) + var oldLogoInput = Files.newInputStream(oldLogo.getPath()) ) { contentEquals = IOUtils.contentEquals(newLogoInput, oldLogoInput); } catch (IOException e) { @@ -268,12 +268,6 @@ public ResultJson updateNamespaceDetails(NamespaceDetailsJson details) { } } - try { - Files.delete(oldLogo); - } catch (IOException e) { - throw new RuntimeException(e); - } - return ResultJson.success("Updated details for namespace " + details.name); } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java index 633e5a92b..f0659697c 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java @@ -41,9 +41,11 @@ public void run(MigrationJobRequest jobRequest) throws Exception { service.deleteResources(extVersion); var entry = migrations.getDownload(extVersion); - var extensionFile = migrations.getExtensionFile(entry); var download = entry.getKey(); - try(var extProcessor = new ExtensionProcessor(extensionFile)) { + try( + var extensionFile = migrations.getExtensionFile(entry); + var extProcessor = new ExtensionProcessor(extensionFile); + ) { extProcessor.processEachResource(download.getExtension(), (resource) -> { resource.setStorageType(download.getStorageType()); migrations.uploadFileResource(resource); @@ -52,6 +54,5 @@ public void run(MigrationJobRequest jobRequest) throws Exception { } service.deleteWebResources(extVersion); - Files.delete(extensionFile); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java index 848d38eaf..dbd23272c 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java @@ -20,7 +20,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import java.nio.file.Files; import java.util.AbstractMap; @Component @@ -46,14 +45,14 @@ public void run(MigrationJobRequest jobRequest) throws Exception { } var content = migrations.getContent(download); - var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); - try(var extProcessor = new ExtensionProcessor(extensionFile)) { + try( + var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); + var extProcessor = new ExtensionProcessor(extensionFile) + ) { var vsixManifest = extProcessor.getVsixManifest(extVersion); vsixManifest.setStorageType(download.getStorageType()); migrations.uploadFileResource(vsixManifest); migrations.persistFileResource(vsixManifest); } - - Files.delete(extensionFile); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java index 91687fa8b..40f5e9d17 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java @@ -49,22 +49,20 @@ public void run(MigrationJobRequest jobRequest) throws Exception { var download = migrations.getResource(jobRequest); var extVersion = download.getExtension(); var content = migrations.getContent(download); - var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); - - boolean fixTargetPlatform; - try(var extProcessor = new ExtensionProcessor(extensionFile)) { - fixTargetPlatform = !extProcessor.getMetadata().getTargetPlatform().equals(extVersion.getTargetPlatform()); - } + try (var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content))) { + boolean fixTargetPlatform; + try (var extProcessor = new ExtensionProcessor(extensionFile)) { + fixTargetPlatform = !extProcessor.getMetadata().getTargetPlatform().equals(extVersion.getTargetPlatform()); + } - if(fixTargetPlatform) { - logger.info("Fixing target platform for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform()); - deleteExtension(extVersion); - try (var input = Files.newInputStream(extensionFile)) { - extensions.publishVersion(input, extVersion.getPublishedWith()); + if (fixTargetPlatform) { + logger.info("Fixing target platform for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform()); + deleteExtension(extVersion); + try (var input = Files.newInputStream(extensionFile.getPath())) { + extensions.publishVersion(input, extVersion.getPublishedWith()); + } } } - - Files.delete(extensionFile); } private void deleteExtension(ExtensionVersion extVersion) { diff --git a/server/src/main/java/org/eclipse/openvsx/migration/GenerateSha256ChecksumJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/GenerateSha256ChecksumJobRequestHandler.java index e2502a5a2..c2dd34bf4 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/GenerateSha256ChecksumJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/GenerateSha256ChecksumJobRequestHandler.java @@ -43,8 +43,10 @@ public void run(MigrationJobRequest jobRequest) throws Exception { } var content = migrations.getContent(download); - var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); - try(var extProcessor = new ExtensionProcessor(extensionFile)) { + try( + var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); + var extProcessor = new ExtensionProcessor(extensionFile) + ) { var checksum = extProcessor.generateSha256Checksum(extVersion); checksum.setStorageType(download.getStorageType()); migrations.uploadFileResource(checksum); diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java index 3560d717f..6f77a686d 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java @@ -16,6 +16,7 @@ import org.eclipse.openvsx.storage.AzureBlobStorageService; import org.eclipse.openvsx.storage.GoogleCloudStorageService; import org.eclipse.openvsx.storage.IStorageService; +import org.eclipse.openvsx.util.TempFile; import org.jobrunr.jobs.lambdas.JobRequestHandler; import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; @@ -29,7 +30,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.util.AbstractMap; import java.util.Map; import java.util.UUID; @@ -73,13 +73,8 @@ public FileResource getResource(MigrationJobRequest jobRequest) { } @Retryable - public Path getExtensionFile(Map.Entry entry) { - Path extensionFile; - try { - extensionFile = Files.createTempFile("extension_", ".vsix"); - } catch (IOException e) { - throw new RuntimeException("Failed to create extension file", e); - } + public TempFile getExtensionFile(Map.Entry entry) throws IOException { + var extensionFile = new TempFile("migration-extension_", ".vsix"); var content = entry.getValue(); if(content == null) { @@ -87,18 +82,14 @@ public Path getExtensionFile(Map.Entry entry) { var storage = getStorage(download); var uri = storage.getLocation(download); backgroundRestTemplate.execute("{extensionLocation}", HttpMethod.GET, null, response -> { - try(var out = Files.newOutputStream(extensionFile)) { + try(var out = Files.newOutputStream(extensionFile.getPath())) { response.getBody().transferTo(out); } return extensionFile; }, Map.of("extensionLocation", uri.toString())); } else { - try { - Files.write(extensionFile, content); - } catch (IOException e) { - throw new RuntimeException("Failed to write to extension file", e); - } + Files.write(extensionFile.getPath(), content); } return extensionFile; @@ -116,7 +107,7 @@ public void uploadFileResource(FileResource resource) { } @Retryable - public void uploadFileResource(FileResource resource, Path extensionFile) { + public void uploadFileResource(FileResource resource, TempFile extensionFile) { if(resource.getStorageType().equals(FileResource.STORAGE_DB)) { return; } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java index 7d8cb7995..cb92bd4e2 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java @@ -42,14 +42,15 @@ public void run(MigrationJobRequest jobRequest) throws Exception { logger.info("Renaming download {}", download.getName()); var content = migrations.getContent(download); - var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); - var newDownload = service.cloneResource(download, name); - migrations.uploadFileResource(newDownload, extensionFile); - migrations.removeFile(download); - - download.setName(name); - service.updateResource(download); - Files.delete(extensionFile); + try(var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content))) { + var newDownload = service.cloneResource(download, name); + migrations.uploadFileResource(newDownload, extensionFile); + migrations.removeFile(download); + + download.setName(name); + service.updateResource(download); + } + logger.info("Updated download name to: {}", name); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java index 08d17a58b..58dbed224 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java @@ -18,14 +18,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import java.nio.file.Files; - @Component @ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) public class SetPreReleaseJobRequestHandler implements JobRequestHandler { protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class)); + @Autowired + MigrationService migrations; + @Autowired SetPreReleaseJobService service; @@ -34,10 +35,10 @@ public class SetPreReleaseJobRequestHandler implements JobRequestHandler getExtensionVersions(MigrationJobRequest jobRequest, Logger logger) { var extension = entityManager.find(Extension.class, jobRequest.getEntityId()); @@ -59,46 +35,7 @@ public List getExtensionVersions(MigrationJobRequest jobReques } @Transactional - public Map.Entry getDownload(ExtensionVersion extVersion) { - var download = repositories.findFileByType(extVersion, FileResource.DOWNLOAD); - var content = download.getStorageType().equals(FileResource.STORAGE_DB) ? download.getContent() : null; - return new AbstractMap.SimpleEntry<>(download, content); - } - - @Retryable - public Path getExtensionFile(Map.Entry entry) { - Path extensionFile; - try { - extensionFile = Files.createTempFile("extension_", ".vsix"); - } catch (IOException e) { - throw new RuntimeException("Failed to create extension file", e); - } - - var content = entry.getValue(); - if(content == null) { - var download = entry.getKey(); - var storage = getStorage(download); - var uri = storage.getLocation(download); - restTemplate.execute("{extensionLocation}", HttpMethod.GET, null, response -> { - try(var out = Files.newOutputStream(extensionFile)) { - response.getBody().transferTo(out); - } - - return extensionFile; - }, Map.of("extensionLocation", uri.toString())); - } else { - try { - Files.write(extensionFile, content); - } catch (IOException e) { - throw new RuntimeException("Failed to write to extension file", e); - } - } - - return extensionFile; - } - - @Transactional - public void updatePreviewAndPreRelease(ExtensionVersion extVersion, Path extensionFile) { + public void updatePreviewAndPreRelease(ExtensionVersion extVersion, TempFile extensionFile) { try(var extProcessor = new ExtensionProcessor(extensionFile)) { extVersion.setPreRelease(extProcessor.isPreRelease()); extVersion.setPreview(extProcessor.isPreview()); @@ -106,13 +43,4 @@ public void updatePreviewAndPreRelease(ExtensionVersion extVersion, Path extensi entityManager.merge(extVersion); } - - private IStorageService getStorage(FileResource resource) { - var storages = Map.of( - FileResource.STORAGE_AZURE, azureStorage, - FileResource.STORAGE_GOOGLE, googleStorage - ); - - return storages.get(resource.getStorageType()); - } } diff --git a/server/src/main/java/org/eclipse/openvsx/mirror/MirrorExtensionService.java b/server/src/main/java/org/eclipse/openvsx/mirror/MirrorExtensionService.java index 6dc77db7e..cf3a4d7dd 100644 --- a/server/src/main/java/org/eclipse/openvsx/mirror/MirrorExtensionService.java +++ b/server/src/main/java/org/eclipse/openvsx/mirror/MirrorExtensionService.java @@ -9,28 +9,14 @@ * ****************************************************************************** */ package org.eclipse.openvsx.mirror; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - import org.eclipse.openvsx.ExtensionService; -import org.eclipse.openvsx.IExtensionRegistry; import org.eclipse.openvsx.UpstreamRegistryService; import org.eclipse.openvsx.UserService; import org.eclipse.openvsx.entities.UserData; import org.eclipse.openvsx.json.ExtensionJson; import org.eclipse.openvsx.json.UserJson; import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.util.NotFoundException; -import org.eclipse.openvsx.util.TargetPlatform; -import org.eclipse.openvsx.util.TimeUtil; -import org.eclipse.openvsx.util.VersionAlias; +import org.eclipse.openvsx.util.*; import org.jobrunr.jobs.context.JobContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +25,15 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; +import java.io.IOException; +import java.nio.file.Files; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + @Component public class MirrorExtensionService { @@ -165,22 +160,15 @@ private void mirrorExtensionVersion(ExtensionJson json) throws RuntimeException throw new RuntimeException("Invalid vsix filename from redirected vsix url"); } - Path extensionFile; - try { - extensionFile = Files.createTempFile("extension_", ".vsix"); - } catch (IOException e) { - throw new RuntimeException("Failed to create extension file", e); - } - - backgroundRestTemplate.execute("{vsixLocation}", HttpMethod.GET, null, response -> { - try(var out = Files.newOutputStream(extensionFile)) { - response.getBody().transferTo(out); - } + try (var extensionFile = new TempFile("extension_", ".vsix")) { + backgroundRestTemplate.execute("{vsixLocation}", HttpMethod.GET, null, response -> { + try(var out = Files.newOutputStream(extensionFile.getPath())) { + response.getBody().transferTo(out); + } - return extensionFile; - }, Map.of("vsixLocation", download)); + return extensionFile; + }, Map.of("vsixLocation", download)); - try { var user = data.getOrAddUser(userJson); var namespace = repositories.findNamespace(namespaceName); data.ensureNamespaceMembership(user, namespace); @@ -191,12 +179,8 @@ private void mirrorExtensionVersion(ExtensionJson json) throws RuntimeException var token = users.useAccessToken(accessTokenValue); extensions.mirrorVersion(extensionFile, token, filename, json.timestamp); logger.debug("completed mirroring of extension version: {}", json.namespace + "." + json.name + "-" + json.version + "@" + json.targetPlatform); - } finally { - try { - Files.delete(extensionFile); - } catch (IOException e) { - logger.error("failed to delete temp file", e); - } + } catch (IOException e) { + throw new RuntimeException(e); } } diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java index 5e693c4e4..e1c138a16 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java @@ -19,6 +19,7 @@ import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TargetPlatform; +import org.eclipse.openvsx.util.TempFile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,8 +30,6 @@ import javax.persistence.EntityManager; import javax.transaction.Transactional; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; @@ -193,7 +192,7 @@ private List updateExistingPublicIds(Extension extension) { @Async @Retryable - public void publishAsync(FileResource download, Path extensionFile, ExtensionService extensionService) { + public void publishAsync(FileResource download, TempFile extensionFile, ExtensionService extensionService) { var extVersion = download.getExtension(); // Delete file resources in case publishAsync is retried service.deleteFileResources(extVersion); @@ -215,13 +214,13 @@ public void publishAsync(FileResource download, Path extensionFile, ExtensionSer // Update whether extension is active, the search index and evict cache service.activateExtension(extVersion, extensionService); try { - Files.delete(extensionFile); + extensionFile.close(); } catch (IOException e) { logger.error("failed to delete temp file", e); } } - public void mirror(FileResource download, Path extensionFile) { + public void mirror(FileResource download, TempFile extensionFile) { var extVersion = download.getExtension(); service.mirrorResource(download); try(var processor = new ExtensionProcessor(extensionFile)) { diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java index 9576e74fb..bd6c42638 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java @@ -9,13 +9,6 @@ * ****************************************************************************** */ package org.eclipse.openvsx.publish; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.persistence.EntityManager; -import javax.transaction.Transactional; - import org.eclipse.openvsx.ExtensionService; import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; @@ -23,10 +16,16 @@ import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.ErrorResultException; +import org.eclipse.openvsx.util.TempFile; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Component; +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.io.IOException; +import java.nio.file.Files; + @Component public class PublishExtensionVersionService { @@ -44,12 +43,12 @@ public void deleteFileResources(ExtensionVersion extVersion) { repositories.findFiles(extVersion).forEach(entityManager::remove); } - public void storeDownload(FileResource download, Path extensionFile) { + public void storeDownload(FileResource download, TempFile extensionFile) { if (storageUtil.shouldStoreExternally(download)) { storageUtil.uploadFile(download, extensionFile); } else { try { - download.setContent(Files.readAllBytes(extensionFile)); + download.setContent(Files.readAllBytes(extensionFile.getPath())); } catch (IOException e) { throw new ErrorResultException("Failed to read extension file", e); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index 11eb9e845..00c3d347e 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -21,6 +21,7 @@ import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.entities.Namespace; import org.eclipse.openvsx.util.TargetPlatform; +import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.util.Pair; @@ -30,8 +31,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -103,12 +102,12 @@ protected void uploadFile(byte[] content, String fileName, String blobName) { } @Override - public void uploadFile(FileResource resource, Path filePath) { + public void uploadFile(FileResource resource, TempFile file) { var blobName = getBlobName(resource); - uploadFile(filePath, resource.getName(), blobName); + uploadFile(file, resource.getName(), blobName); } - protected void uploadFile(Path filePath, String fileName, String blobName) { + protected void uploadFile(TempFile file, String fileName, String blobName) { if (Strings.isNullOrEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot upload file " + blobName + ": missing Azure blob service endpoint"); @@ -124,7 +123,7 @@ protected void uploadFile(Path filePath, String fileName, String blobName) { headers.setCacheControl(cacheControl.getHeaderValue()); } - blobClient.uploadFromFile(filePath.toAbsolutePath().toString(), true); + blobClient.uploadFromFile(file.getPath().toAbsolutePath().toString(), true); blobClient.setHttpHeaders(headers); } @@ -200,14 +199,10 @@ protected String getBlobName(Namespace namespace) { } @Override - public Path downloadNamespaceLogo(Namespace namespace) { - try { - var logoFile = Files.createTempFile("namespace-logo", ".png"); - getContainerClient().getBlobClient(getBlobName(namespace)).downloadToFile(logoFile.toString(), true); - return logoFile; - } catch (IOException e) { - throw new RuntimeException(e); - } + public TempFile downloadNamespaceLogo(Namespace namespace) throws IOException { + var logoFile = new TempFile("namespace-logo", ".png"); + getContainerClient().getBlobClient(getBlobName(namespace)).downloadToFile(logoFile.getPath().toString(), true); + return logoFile; } @Override diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java index f4e03277d..b5163d112 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; +import org.eclipse.openvsx.util.TempFile; import org.jobrunr.jobs.annotations.Job; import org.jobrunr.spring.annotations.Recurring; import org.slf4j.Logger; @@ -33,7 +34,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -150,15 +150,10 @@ public void updateDownloadCounts() { } private Map> processBlobItem(String blobName) { - Path downloadsTempFile; - try { - downloadsTempFile = Files.createTempFile("azure-downloads-", ".json"); - } catch (IOException e) { - throw new RuntimeException(e); - } - - getContainerClient().getBlobClient(blobName).downloadToFile(downloadsTempFile.toAbsolutePath().toString(), true); - try (var reader = Files.newBufferedReader(downloadsTempFile)) { + try ( + var downloadsTempFile = downloadBlobItem(blobName); + var reader = Files.newBufferedReader(downloadsTempFile.getPath()) + ) { return reader.lines() .map(line -> { try { @@ -187,15 +182,15 @@ private Map> processBlobItem(String blobName) { .collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); } catch (IOException e) { throw new RuntimeException(e); - } finally { - try { - Files.delete(downloadsTempFile); - } catch (IOException e) { - logger.error("Failed to delete downloads file", e); - } } } + private TempFile downloadBlobItem(String blobName) throws IOException { + var downloadsTempFile = new TempFile("azure-downloads-", ".json"); + getContainerClient().getBlobClient(blobName).downloadToFile(downloadsTempFile.getPath().toAbsolutePath().toString(), true); + return downloadsTempFile; + } + private List getBlobNames(List items) { var blobNames = new ArrayList(); for(var item : items) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index 7879946b8..36196b965 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -18,6 +18,7 @@ import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.entities.Namespace; import org.eclipse.openvsx.util.TargetPlatform; +import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.util.Pair; @@ -28,7 +29,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; @Component @@ -99,17 +99,17 @@ protected void uploadFile(byte[] content, String fileName, String objectId) { } @Override - public void uploadFile(FileResource resource, Path filePath) { + public void uploadFile(FileResource resource, TempFile file) { var objectId = getObjectId(resource); if (Strings.isNullOrEmpty(bucketId)) { throw new IllegalStateException("Cannot upload file " + objectId + ": missing Google bucket id"); } - uploadFile(filePath, resource.getName(), objectId); + uploadFile(file, resource.getName(), objectId); } - protected void uploadFile(Path filePath, String fileName, String objectId) { + protected void uploadFile(TempFile file, String fileName, String objectId) { var blobInfoBuilder = BlobInfo.newBuilder(BlobId.of(bucketId, objectId)) .setContentType(StorageUtil.getFileType(fileName).toString()); if (fileName.endsWith(".vsix")) { @@ -119,7 +119,7 @@ protected void uploadFile(Path filePath, String fileName, String objectId) { blobInfoBuilder.setCacheControl(cacheControl.getHeaderValue()); } try ( - var in = Files.newByteChannel(filePath); + var in = Files.newByteChannel(file.getPath()); var out = getStorage().writer(blobInfoBuilder.build()) ) { var buffer = ByteBuffer.allocateDirect(1024 * 1024); @@ -189,21 +189,13 @@ protected String getObjectId(Namespace namespace) { } @Override - public Path downloadNamespaceLogo(Namespace namespace) { - Path logoFile; - try { - logoFile = Files.createTempFile("namespace-logo", ".png"); - } catch (IOException e) { - throw new RuntimeException(e); - } - + public TempFile downloadNamespaceLogo(Namespace namespace) throws IOException { + var logoFile = new TempFile("namespace-logo", ".png"); try ( var reader = getStorage().reader(BlobId.of(bucketId, getObjectId(namespace))); - var output = new FileOutputStream(logoFile.toFile()); + var output = new FileOutputStream(logoFile.getPath().toFile()) ) { output.getChannel().transferFrom(reader, 0, Long.MAX_VALUE); - } catch (IOException e) { - throw new RuntimeException(e); } return logoFile; diff --git a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java index 36329cb46..01a09aff2 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java @@ -9,14 +9,15 @@ ********************************************************************************/ package org.eclipse.openvsx.storage; -import java.net.URI; -import java.nio.file.Path; -import java.util.List; - import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.util.TempFile; import org.springframework.data.util.Pair; +import java.io.IOException; +import java.net.URI; +import java.util.List; + public interface IStorageService { /** @@ -32,7 +33,7 @@ public interface IStorageService { /** * Upload a file to the external storage. */ - void uploadFile(FileResource resource, Path filePath); + void uploadFile(FileResource resource, TempFile file); /** * Remove a file from the external storage. @@ -59,7 +60,7 @@ public interface IStorageService { */ URI getNamespaceLogoLocation(Namespace namespace); - Path downloadNamespaceLogo(Namespace namespace); + TempFile downloadNamespaceLogo(Namespace namespace) throws IOException; void copyFiles(List> pairs); } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 1affe00cc..82f7335d2 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -18,6 +18,7 @@ import org.eclipse.openvsx.entities.Namespace; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; +import org.eclipse.openvsx.util.TempFile; import org.eclipse.openvsx.util.TimeUtil; import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -34,7 +35,6 @@ import java.io.IOException; import java.net.URI; import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -138,14 +138,14 @@ public void uploadFile(FileResource resource) { } @Override - public void uploadFile(FileResource resource, Path filePath) { + public void uploadFile(FileResource resource, TempFile file) { var storageType = getActiveStorageType(); switch (storageType) { case STORAGE_GOOGLE: - googleStorage.uploadFile(resource, filePath); + googleStorage.uploadFile(resource, file); break; case STORAGE_AZURE: - azureStorage.uploadFile(resource, filePath); + azureStorage.uploadFile(resource, file); break; default: throw new RuntimeException("External storage is not available."); @@ -224,7 +224,7 @@ public URI getNamespaceLogoLocation(Namespace namespace) { } } - public Path downloadNamespaceLogo(Namespace namespace) { + public TempFile downloadNamespaceLogo(Namespace namespace) throws IOException { if(namespace.getLogoStorageType() == null) { return createNamespaceLogoFile(); } @@ -235,24 +235,16 @@ public Path downloadNamespaceLogo(Namespace namespace) { case STORAGE_AZURE: return azureStorage.downloadNamespaceLogo(namespace); case STORAGE_DB: - try { - var logoFile = createNamespaceLogoFile(); - Files.write(logoFile, namespace.getLogoBytes()); - return logoFile; - } catch (IOException e) { - throw new RuntimeException(e); - } + var logoFile = createNamespaceLogoFile(); + Files.write(logoFile.getPath(), namespace.getLogoBytes()); + return logoFile; default: return createNamespaceLogoFile(); } } - private Path createNamespaceLogoFile() { - try { - return Files.createTempFile("namespace-logo", ".png"); - } catch (IOException e) { - throw new RuntimeException(e); - } + private TempFile createNamespaceLogoFile() throws IOException { + return new TempFile("namespace-logo", ".png"); } private String getFileUrl(String name, ExtensionVersion extVersion, String serverUrl) { diff --git a/server/src/main/java/org/eclipse/openvsx/util/TempFile.java b/server/src/main/java/org/eclipse/openvsx/util/TempFile.java new file mode 100644 index 000000000..fd80efede --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/util/TempFile.java @@ -0,0 +1,32 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.util; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class TempFile implements AutoCloseable { + + private final Path path; + + public TempFile(String prefix, String suffix) throws IOException { + path = Files.createTempFile(prefix, suffix); + } + + public Path getPath() { + return path; + } + + @Override + public void close() throws IOException { + Files.delete(path); + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java b/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java index ffaf7bd4e..cdff3e6e3 100644 --- a/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java +++ b/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java @@ -9,23 +9,24 @@ ********************************************************************************/ package org.eclipse.openvsx; -import static org.assertj.core.api.Assertions.assertThat; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.util.TempFile; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.util.Arrays; -import java.util.Collections; -import org.eclipse.openvsx.entities.FileResource; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; class ExtensionProcessorTest { @Test void testTodoTree() throws Exception { - var path = writeToTempFile("util/todo-tree.zip"); - try (var processor = new ExtensionProcessor(path)) { + try ( + var file = writeToTempFile("util/todo-tree.zip"); + var processor = new ExtensionProcessor(file) + ) { assertThat(processor.getNamespace()).isEqualTo("Gruntfuggly"); assertThat(processor.getExtensionName()).isEqualTo("todo-tree"); @@ -42,55 +43,53 @@ void testTodoTree() throws Exception { checkResource(processor, FileResource.README, "README.md"); checkResource(processor, FileResource.ICON, "todo-tree.png"); checkResource(processor, FileResource.LICENSE, "License.txt"); - } finally { - Files.delete(path); } } @Test void testChangelog() throws Exception { - var path = writeToTempFile("util/changelog.zip"); - try (var processor = new ExtensionProcessor(path)) { + try ( + var file = writeToTempFile("util/changelog.zip"); + var processor = new ExtensionProcessor(file) + ) { checkResource(processor, FileResource.CHANGELOG, "CHANGELOG.md"); - } finally { - Files.delete(path); } } @Test void testCapitalizedCaseForResources() throws Exception { - var path = writeToTempFile("util/with-capitalized-case.zip"); - try (var processor = new ExtensionProcessor(path)) { + try ( + var file = writeToTempFile("util/with-capitalized-case.zip"); + var processor = new ExtensionProcessor(file) + ) { checkResource(processor, FileResource.CHANGELOG, "Changelog.md"); checkResource(processor, FileResource.README, "Readme.md"); checkResource(processor, FileResource.LICENSE, "License.txt"); - } finally { - Files.delete(path); } } @Test void testMinorCaseForResources() throws Exception { - var path = writeToTempFile("util/with-minor-case.zip"); - try (var processor = new ExtensionProcessor(path)) { + try ( + var file = writeToTempFile("util/with-minor-case.zip"); + var processor = new ExtensionProcessor(file) + ) { checkResource(processor, FileResource.CHANGELOG, "changelog.md"); checkResource(processor, FileResource.README, "readme.md"); checkResource(processor, FileResource.LICENSE, "license.txt"); - } finally { - Files.delete(path); } } - private Path writeToTempFile(String resource) throws IOException { - var path = Files.createTempFile("test", ".zip"); + private TempFile writeToTempFile(String resource) throws IOException { + var file = new TempFile("test", ".zip"); try( var in = getClass().getResourceAsStream(resource); - var out = Files.newOutputStream(path); + var out = Files.newOutputStream(file.getPath()); ) { in.transferTo(out); } - return path; + return file; } private void checkResource(ExtensionProcessor processor, String type, String expectedName) {