Use [Content_Types].xml to set FileResource contentType

eclipse · Jun 10, 2022 · 954393d · 954393d
1 parent fb56cee
commit 954393d
Show file tree

Hide file tree

Showing 15 changed files with 154 additions and 80 deletions.
diff --git a/server/build.gradle b/server/build.gradle
@@ -29,8 +29,7 @@ def versions = [
     junit: '5.7.1',
     testcontainers: '1.15.2',
     jackson: '2.12.5',
-    woodstox: '6.2.4',
-    tika: '2.4.0'
+    woodstox: '6.2.4'
 ]
 ext['junit-jupiter.version'] = versions.junit
 sourceCompatibility = versions.java
@@ -67,7 +66,6 @@ dependencies {
     implementation "org.springframework.security:spring-security-oauth2-jose"
     implementation "org.springframework.session:spring-session-jdbc"
     implementation "org.flywaydb:flyway-core:${versions.flyway}"
-    implementation "org.apache.tika:tika-core:${versions.tika}"
     implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}"
     implementation "com.azure:azure-storage-blob:${versions.azure}"
     implementation "io.springfox:springfox-boot-starter:${versions.springfox}"

diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java
@@ -9,10 +9,8 @@
  ********************************************************************************/
 package org.eclipse.openvsx;
 
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -38,6 +36,11 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.data.util.Pair;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 
 /**
  * Processes uploaded extension files and extracts their metadata.
@@ -291,6 +294,8 @@ private List<String> getEngines(JsonNode node) {
 
     public List<FileResource> getResources(ExtensionVersion extension) {
         var resources = new ArrayList<>(getAllResources(extension));
+        var contentTypes = loadContentTypes(resources);
+
         var binary = getBinary(extension);
         if (binary != null)
             resources.add(binary);
@@ -310,7 +315,9 @@ public List<FileResource> getResources(ExtensionVersion extension) {
         if (icon != null)
             resources.add(icon);
 
-        return resources;
+        return resources.stream()
+                .map(resource -> setContentType(resource, contentTypes))
+                .collect(Collectors.toList());
     }
 
     protected List<FileResource> getAllResources(ExtensionVersion extension) {
@@ -421,6 +428,48 @@ protected FileResource getLicense(ExtensionVersion extension) {
         return license;
     }
 
+    private Map<String, String> loadContentTypes(List<FileResource> resources) {
+        var contentTypes = resources.stream()
+                .filter(r -> r.getName().equals("[Content_Types].xml"))
+                .findFirst()
+                .map(FileResource::getContent)
+                .map(this::parseContentTypesXml)
+                .orElse(new HashMap<>());
+
+        contentTypes.putIfAbsent(".vsix", "application/zip");
+        return contentTypes;
+    }
+
+    private Map<String, String> parseContentTypesXml(byte[] content) {
+        try (var input = new ByteArrayInputStream(content)) {
+            var contentTypes = new HashMap<String, String>();
+            var document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(input);
+            var elements = document.getDocumentElement().getElementsByTagName("Default");
+            for(var i = 0; i < elements.getLength(); i++) {
+                var element = elements.item(i);
+                var attributes = element.getAttributes();
+                var extension = attributes.getNamedItem("Extension").getTextContent();
+                var contentType = attributes.getNamedItem("ContentType").getTextContent();
+                contentTypes.put(extension, contentType);
+            }
+
+            return contentTypes;
+        } catch (IOException | ParserConfigurationException | SAXException e) {
+            logger.error("failed to read content types", e);
+            return null;
+        }
+    }
+
+    private FileResource setContentType(FileResource resource, Map<String, String> contentTypes) {
+        var resourceName = Optional.ofNullable(resource.getName()).orElse("");
+        var fileExtensionIndex = resourceName.lastIndexOf('.');
+        var fileExtension = fileExtensionIndex != -1 ? resourceName.substring(fileExtensionIndex) : "";
+        var contentType = contentTypes.getOrDefault(fileExtension, MediaType.APPLICATION_OCTET_STREAM_VALUE);
+
+        resource.setContentType(contentType);
+        return resource;
+    }
+
     private void detectLicense(byte[] content, ExtensionVersion extension) {
         if (Strings.isNullOrEmpty(extension.getLicense())) {
             var detection = new LicenseDetection();

diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java
@@ -165,7 +165,7 @@ public ResponseEntity<byte[]> getFile(String namespace, String extensionName, St
         if (resource.getType().equals(DOWNLOAD))
             storageUtil.increaseDownloadCount(extVersion, resource);
         if (resource.getStorageType().equals(FileResource.STORAGE_DB)) {
-            var headers = storageUtil.getFileResponseHeaders(fileName);
+            var headers = storageUtil.getFileResponseHeaders(resource);
             return new ResponseEntity<>(resource.getContent(), headers, HttpStatus.OK);
         } else {
             return ResponseEntity.status(HttpStatus.FOUND)

diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeAdapter.java
@@ -309,7 +309,7 @@ public ResponseEntity<byte[]> getAsset(HttpServletRequest request,
         if (resource.getType().equals(FileResource.DOWNLOAD))
             storageUtil.increaseDownloadCount(extVersion, resource);
         if (resource.getStorageType().equals(FileResource.STORAGE_DB)) {
-            var headers = storageUtil.getFileResponseHeaders(resource.getName());
+            var headers = storageUtil.getFileResponseHeaders(resource);
             return new ResponseEntity<>(resource.getContent(), headers, HttpStatus.OK);
         } else {
             return ResponseEntity.status(HttpStatus.FOUND)
@@ -408,7 +408,7 @@ private ResponseEntity<byte[]> browseFile(
             String version
     ) {
         if (resource.getStorageType().equals(FileResource.STORAGE_DB)) {
-            var headers = storageUtil.getFileResponseHeaders(resource.getName());
+            var headers = storageUtil.getFileResponseHeaders(resource);
             return new ResponseEntity<>(resource.getContent(), headers, HttpStatus.OK);
         } else {
             var namespace = new Namespace();

diff --git a/server/src/main/java/org/eclipse/openvsx/dto/FileResourceDTO.java b/server/src/main/java/org/eclipse/openvsx/dto/FileResourceDTO.java
@@ -11,8 +11,6 @@
 
 import org.eclipse.openvsx.entities.FileResource;
 
-import java.util.Objects;
-
 public class FileResourceDTO {
 
     private final long extensionVersionId;
@@ -23,11 +21,21 @@ public class FileResourceDTO {
     private final String type;
     private String storageType;
     private byte[] content;
-
-    public FileResourceDTO(long id, long extensionVersionId, String name, String type, String storageType, byte[] content) {
+    private String contentType;
+
+    public FileResourceDTO(
+            long id,
+            long extensionVersionId,
+            String name,
+            String type,
+            String storageType,
+            byte[] content,
+            String contentType
+    ) {
         this(id, extensionVersionId, name, type);
         this.storageType = storageType;
         this.content = content;
+        this.contentType = contentType;
     }
 
     public FileResourceDTO(long id, long extensionVersionId, String name, String type) {
@@ -73,6 +81,8 @@ public byte[] getContent() {
         return content;
     }
 
+    public String getContentType() { return contentType; }
+
     public boolean isWebResource() {
         return type.equals(FileResource.RESOURCE) && name.startsWith("extension/");
     }

diff --git a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java
@@ -46,6 +46,8 @@ public class FileResource {
 
     byte[] content;
 
+    String contentType;
+
     @Column(length = 32)
     String storageType;
 
@@ -90,6 +92,14 @@ public void setContent(byte[] content) {
 		this.content = content;
 	}
 
+	public String getContentType() {
+        return contentType;
+    }
+
+    public void setContentType(String contentType) {
+        this.contentType = contentType;
+    }
+
     public String getStorageType() {
         return storageType;
     }

diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceDTORepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceDTORepository.java
@@ -30,15 +30,28 @@ public class FileResourceDTORepository {
     DSLContext dsl;
 
     public List<FileResourceDTO> findAll(Collection<Long> extensionIds, Collection<String> types) {
-        return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE)
+        return dsl.select(
+                    FILE_RESOURCE.ID,
+                    FILE_RESOURCE.EXTENSION_ID,
+                    FILE_RESOURCE.NAME,
+                    FILE_RESOURCE.TYPE
+                )
                 .from(FILE_RESOURCE)
                 .where(FILE_RESOURCE.EXTENSION_ID.in(extensionIds))
                 .and(FILE_RESOURCE.TYPE.in(types))
                 .fetchInto(FileResourceDTO.class);
     }
 
     public List<FileResourceDTO> findAllResources(String namespaceName, String extensionName, String version, String prefix) {
-        return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE, FILE_RESOURCE.STORAGE_TYPE, FILE_RESOURCE.CONTENT)
+        return dsl.select(
+                    FILE_RESOURCE.ID,
+                    FILE_RESOURCE.EXTENSION_ID,
+                    FILE_RESOURCE.NAME,
+                    FILE_RESOURCE.TYPE,
+                    FILE_RESOURCE.STORAGE_TYPE,
+                    FILE_RESOURCE.CONTENT,
+                    FILE_RESOURCE.CONTENT_TYPE
+                )
                 .from(FILE_RESOURCE)
                 .join(EXTENSION_VERSION).on(EXTENSION_VERSION.ID.eq(FILE_RESOURCE.EXTENSION_ID))
                 .join(EXTENSION).on(EXTENSION.ID.eq(EXTENSION_VERSION.EXTENSION_ID))

diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java
@@ -64,19 +64,21 @@ public void uploadFile(FileResource resource) {
                     + blobName + ": missing Azure blob service endpoint");
         }
 
-        uploadFile(resource.getContent(), resource.getName(), blobName);
+        uploadFile(resource, blobName);
     }
 
-    protected void uploadFile(byte[] content, String fileName, String blobName) {
+    protected void uploadFile(FileResource resource, String blobName) {
         var blobClient = getContainerClient().getBlobClient(blobName);
         var headers = new BlobHttpHeaders();
-        headers.setContentType(StorageUtil.getFileType(fileName).toString());
-        if (fileName.endsWith(".vsix")) {
-            headers.setContentDisposition("attachment; filename=\"" + fileName + "\"");
+        headers.setContentType(resource.getContentType());
+        if (resource.getName().endsWith(".vsix")) {
+            headers.setContentDisposition("attachment; filename=\"" + resource.getName() + "\"");
         } else {
-            var cacheControl = StorageUtil.getCacheControl(fileName);
+            var cacheControl = StorageUtil.getCacheControl(resource.getName());
             headers.setCacheControl(cacheControl.getHeaderValue());
         }
+
+        var content = resource.getContent();
         try (var dataStream = new ByteArrayInputStream(content)) {
             blobClient.upload(dataStream, content.length, true);
             blobClient.setHttpHeaders(headers);

diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java
@@ -64,19 +64,19 @@ public void uploadFile(FileResource resource) {
                     + objectId + ": missing Google bucket id");
         }
 
-        uploadFile(resource.getContent(), resource.getName(), objectId);
+        uploadFile(resource, objectId);
     }
 
-    protected void uploadFile(byte[] content, String fileName, String objectId) {
+    protected void uploadFile(FileResource resource, String objectId) {
         var blobInfoBuilder = BlobInfo.newBuilder(BlobId.of(bucketId, objectId))
-                .setContentType(StorageUtil.getFileType(fileName).toString());
-        if (fileName.endsWith(".vsix")) {
-            blobInfoBuilder.setContentDisposition("attachment; filename=\"" + fileName + "\"");
+                .setContentType(resource.getContentType());
+        if (resource.getName().endsWith(".vsix")) {
+            blobInfoBuilder.setContentDisposition("attachment; filename=\"" + resource.getName() + "\"");
         } else {
-            var cacheControl = StorageUtil.getCacheControl(fileName);
+            var cacheControl = StorageUtil.getCacheControl(resource.getName());
             blobInfoBuilder.setCacheControl(cacheControl.getHeaderValue());
         }
-        getStorage().create(blobInfoBuilder.build(), content);
+        getStorage().create(blobInfoBuilder.build(), resource.getContent());
     }
 
     @Override

diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtil.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtil.java
@@ -10,35 +10,14 @@
 
 package org.eclipse.openvsx.storage;
 
-import org.apache.tika.Tika;
 import org.springframework.http.CacheControl;
-import org.springframework.http.MediaType;
 
-import java.net.URLConnection;
 import java.util.concurrent.TimeUnit;
 
 class StorageUtil {
 
     private StorageUtil(){}
 
-    @Deprecated
-    static MediaType getFileTypeDeprecated(String fileName) {
-        if (fileName.endsWith(".vsix"))
-            return MediaType.APPLICATION_OCTET_STREAM;
-        if (fileName.endsWith(".json"))
-            return MediaType.APPLICATION_JSON;
-        var contentType = URLConnection.guessContentTypeFromName(fileName);
-        if (contentType != null)
-            return MediaType.parseMediaType(contentType);
-        return MediaType.TEXT_PLAIN;
-    }
-
-    static MediaType getFileType(String fileName) {
-        var tika = new Tika();
-        var contentType = tika.detect(fileName); // returns 'application/octet-stream' by default
-        return MediaType.parseMediaType(contentType);
-    }
-
     static CacheControl getCacheControl(String fileName) {
         // Files are requested with a version string in the URL, so their content cannot change
         return CacheControl.maxAge(30, TimeUnit.DAYS).cachePublic();

diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java
@@ -12,18 +12,16 @@
 import static org.eclipse.openvsx.entities.FileResource.*;
 
 import java.net.URI;
-import java.net.URLConnection;
 import java.time.LocalDateTime;
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 
 import javax.persistence.EntityManager;
 import javax.transaction.Transactional;
 
 import com.google.common.base.Strings;
 
-import org.apache.commons.lang3.ArrayUtils;
 import org.eclipse.openvsx.cache.CacheService;
+import org.eclipse.openvsx.dto.FileResourceDTO;
 import org.eclipse.openvsx.entities.Download;
 import org.eclipse.openvsx.entities.ExtensionVersion;
 import org.eclipse.openvsx.entities.FileResource;
@@ -33,7 +31,6 @@
 import org.eclipse.openvsx.util.UrlUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.CacheControl;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Component;
@@ -195,9 +192,17 @@ public void increaseDownloadCount(ExtensionVersion extVersion, FileResource reso
         cache.evictExtensionJsons(extension);
     }
 
-    public HttpHeaders getFileResponseHeaders(String fileName) {
+    public HttpHeaders getFileResponseHeaders(FileResource resource) {
+        return getFileResponseHeaders(resource.getName(), resource.getContentType());
+    }
+
+    public HttpHeaders getFileResponseHeaders(FileResourceDTO resource) {
+        return getFileResponseHeaders(resource.getName(), resource.getContentType());
+    }
+
+    private HttpHeaders getFileResponseHeaders(String fileName, String contentType) {
         var headers = new HttpHeaders();
-        headers.setContentType(StorageUtil.getFileType(fileName));
+        headers.setContentType(MediaType.parseMediaType(contentType));
         if (fileName.endsWith(".vsix")) {
             headers.add("Content-Disposition", "attachment; filename=\"" + fileName + "\"");
         } else {

diff --git a/server/src/main/resources/db/migration/V1_25__FileResource_ContentType.sql b/server/src/main/resources/db/migration/V1_25__FileResource_ContentType.sql
@@ -0,0 +1 @@
+ALTER TABLE file_resource ADD COLUMN content_type CHARACTER VARYING(255);
diff --git a/server/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml b/server/src/main/resources/org/apache/tika/mime/custom-mimetypes.xml