From 024472bc71aa8d3e60fdde2b5bb9f795afe3fb6d Mon Sep 17 00:00:00 2001 From: Emanuel Dima Date: Thu, 4 Jun 2020 08:55:05 +0200 Subject: [PATCH] enable POST calls and streamline popup view --- .../switchboard/app/SwitchboardApp.java | 2 +- .../eu/clarin/switchboard/core/FileInfo.java | 2 +- .../clarin/switchboard/core/MediaLibrary.java | 99 ++++++++++++++++--- .../core/xc/SwitchboardExceptionMapper.java | 4 +- .../switchboard/resources/DataResource.java | 48 ++++++++- .../switchboard/resources/IndexView.java | 31 ++++++ .../switchboard/resources/MainResource.java | 35 +++++-- .../switchboard/resources/index.mustache | 52 +--------- .../webui/popup/switchboardpopup.css | 1 - .../resources/webui/popup/switchboardpopup.js | 19 ++-- webui/src/actions/actions.jsx | 39 ++++++++ webui/src/actions/reducers.jsx | 10 ++ webui/src/components/Alerts.jsx | 9 +- webui/src/components/NavBar.jsx | 25 +++-- webui/src/constants.jsx | 18 ++-- webui/src/containers/AlertsContainer.jsx | 1 + webui/src/index.jsx | 32 ++++-- webui/webpack.config.js | 12 ++- 18 files changed, 321 insertions(+), 118 deletions(-) diff --git a/backend/src/main/java/eu/clarin/switchboard/app/SwitchboardApp.java b/backend/src/main/java/eu/clarin/switchboard/app/SwitchboardApp.java index 429b68952..3f0f33585 100644 --- a/backend/src/main/java/eu/clarin/switchboard/app/SwitchboardApp.java +++ b/backend/src/main/java/eu/clarin/switchboard/app/SwitchboardApp.java @@ -104,7 +104,7 @@ public void run(RootConfig configuration, Environment environment) throws IOExce environment.jersey().register(infoResource); environment.jersey().register(dataResource); environment.jersey().register(toolsResource); - environment.jersey().register(new MainResource()); + environment.jersey().register(new MainResource(mediaLibrary)); environment.healthChecks().register("switchboard", new AppHealthCheck()); } diff --git a/backend/src/main/java/eu/clarin/switchboard/core/FileInfo.java b/backend/src/main/java/eu/clarin/switchboard/core/FileInfo.java index 2d10d25a1..de6d08959 100644 --- a/backend/src/main/java/eu/clarin/switchboard/core/FileInfo.java +++ b/backend/src/main/java/eu/clarin/switchboard/core/FileInfo.java @@ -30,7 +30,7 @@ public FileInfo(UUID id, String filename, Path path) { this.path = path; this.creation = new Date().toInstant(); - this.fileLength = path.toFile().length(); + this.fileLength = path == null ? -1 : path.toFile().length(); } public UUID getId() { diff --git a/backend/src/main/java/eu/clarin/switchboard/core/MediaLibrary.java b/backend/src/main/java/eu/clarin/switchboard/core/MediaLibrary.java index eef27a767..62f15868a 100644 --- a/backend/src/main/java/eu/clarin/switchboard/core/MediaLibrary.java +++ b/backend/src/main/java/eu/clarin/switchboard/core/MediaLibrary.java @@ -1,6 +1,5 @@ package eu.clarin.switchboard.core; -import com.google.common.io.ByteStreams; import eu.clarin.switchboard.app.config.DataStoreConfig; import eu.clarin.switchboard.app.config.UrlResolverConfig; import eu.clarin.switchboard.core.xc.CommonException; @@ -10,7 +9,6 @@ import eu.clarin.switchboard.profiler.api.Profile; import eu.clarin.switchboard.profiler.api.Profiler; import eu.clarin.switchboard.profiler.api.ProfilingException; -import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.cache.CacheConfig; @@ -23,6 +21,7 @@ import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -39,20 +38,38 @@ public class MediaLibrary { private final DataStore dataStore; private final Profiler profiler; private final StoragePolicy storagePolicy; - private final UrlResolverConfig urlResolverConfig; private final CloseableHttpClient cachingClient; + private final ExecutorService executorService; Map fileInfoMap = Collections.synchronizedMap(new HashMap<>()); + public static class FileError { + Instant creation; + Exception exception; + + public FileError(Exception exception) { + this.creation = Instant.now(); + this.exception = exception; + } + + public Instant getCreation() { + return creation; + } + + public Exception getException() { + return exception; + } + } + Map fileInfoAsyncErrorMap = Collections.synchronizedMap(new HashMap<>()); + public MediaLibrary(DataStore dataStore, Profiler profiler, StoragePolicy storagePolicy, - UrlResolverConfig urlResolver, DataStoreConfig dataStoreConfig) { + UrlResolverConfig urlResolverConfig, DataStoreConfig dataStoreConfig) { this.dataStore = dataStore; this.profiler = profiler; this.storagePolicy = storagePolicy; - this.urlResolverConfig = urlResolver; CacheConfig cacheConfig = CacheConfig.custom() - .setMaxCacheEntries(urlResolver.getMaxHttpCacheEntries()) + .setMaxCacheEntries(urlResolverConfig.getMaxHttpCacheEntries()) .setMaxObjectSize(dataStoreConfig.getMaxSize()) .build(); RequestConfig requestConfig = RequestConfig.custom() @@ -66,12 +83,64 @@ public MediaLibrary(DataStore dataStore, Profiler profiler, StoragePolicy storag .setDefaultRequestConfig(requestConfig) .build(); + executorService = Executors.newCachedThreadPool(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); Duration cleanup = storagePolicy.getCleanupPeriod(); executor.scheduleAtFixedRate(this::periodicCleanup, cleanup.getSeconds(), cleanup.getSeconds(), TimeUnit.SECONDS); } public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException, ProfilingException { + UUID id = UUID.randomUUID(); + return addMedia(id, originalUrlOrDoiOrHandle); + } + + public UUID addMediaAsync(String originalUrlOrDoiOrHandle) { + UUID id = UUID.randomUUID(); + fileInfoMap.put(id, new FileInfo(id, null, null)); + executorService.submit(() -> { + try { + addMedia(id, originalUrlOrDoiOrHandle); + } catch (Exception exception) { + LOGGER.debug("async error: {}", exception.getMessage()); + fileInfoAsyncErrorMap.put(id, new FileError(exception)); + fileInfoMap.remove(id); + } + }); + return id; + } + + public FileInfo addMedia(String filename, InputStream inputStream) throws + StoragePolicyException, StorageException, ProfilingException { + UUID id = UUID.randomUUID(); + return addMedia(id, filename, inputStream); + } + + public UUID addMediaAsync(String filename, InputStream inputStream) { + UUID id = UUID.randomUUID(); + fileInfoMap.put(id, new FileInfo(id, null, null)); + executorService.submit(() -> { + try { + addMedia(id, filename, inputStream); + } catch (Exception exception) { + LOGGER.debug("async error: {}", exception.getMessage()); + fileInfoAsyncErrorMap.put(id, new FileError(exception)); + fileInfoMap.remove(id); + } + }); + return id; + } + + public FileInfo getFileInfo(UUID id) { + return fileInfoMap.get(id); + } + + public Exception getFileInfoAsyncError(UUID id) { + FileError fileError = fileInfoAsyncErrorMap.get(id); + return fileError == null ? null : fileError.getException(); + } + + private FileInfo addMedia(UUID id, String originalUrlOrDoiOrHandle) throws CommonException, ProfilingException { LinkMetadata.LinkInfo linkInfo = LinkMetadata.getLinkData(cachingClient, originalUrlOrDoiOrHandle); try { if (linkInfo.response.getEntity().getContentLength() > storagePolicy.getMaxAllowedDataSize()) { @@ -80,7 +149,7 @@ public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException DefaultStoragePolicy.humanSize(storagePolicy.getMaxAllowedDataSize()) + ".", StoragePolicyException.Kind.TOO_BIG); } - FileInfo fileInfo = addMedia(linkInfo.filename, linkInfo.response.getEntity().getContent()); + FileInfo fileInfo = addMedia(id, linkInfo.filename, linkInfo.response.getEntity().getContent()); fileInfo.setLinksInfo(originalUrlOrDoiOrHandle, linkInfo.downloadLink, linkInfo.redirects); return fileInfo; } catch (IOException xc) { @@ -94,9 +163,8 @@ public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException } } - public FileInfo addMedia(String filename, InputStream inputStream) throws + private FileInfo addMedia(UUID id, String filename, InputStream inputStream) throws StoragePolicyException, StorageException, ProfilingException { - UUID id = UUID.randomUUID(); Path path; try { path = dataStore.save(id, filename, inputStream); @@ -138,10 +206,6 @@ public FileInfo addMedia(String filename, InputStream inputStream) throws return fileInfo; } - public FileInfo getFileInfo(UUID id) { - return fileInfoMap.get(id); - } - private void periodicCleanup() { // this runs on its own thread LOGGER.info("start periodic cleanup now"); @@ -154,5 +218,14 @@ private void periodicCleanup() { iterator.remove(); } } + + LOGGER.info("start periodic error cleanup now"); + for (Iterator iterator = fileInfoAsyncErrorMap.values().iterator(); iterator.hasNext(); ) { + FileError fe = iterator.next(); + Duration lifetime = Duration.between(fe.getCreation(), Instant.now()); + if (lifetime.compareTo(storagePolicy.getMaxAllowedLifetime()) > 0) { + iterator.remove(); + } + } } } diff --git a/backend/src/main/java/eu/clarin/switchboard/core/xc/SwitchboardExceptionMapper.java b/backend/src/main/java/eu/clarin/switchboard/core/xc/SwitchboardExceptionMapper.java index 2b04dc021..c57c390ef 100644 --- a/backend/src/main/java/eu/clarin/switchboard/core/xc/SwitchboardExceptionMapper.java +++ b/backend/src/main/java/eu/clarin/switchboard/core/xc/SwitchboardExceptionMapper.java @@ -66,7 +66,7 @@ public Response toResponse(Exception exception) { return Response.status(status).entity(json).build(); } - public Response toResponse(StoragePolicyException exception) { + private static Response toResponse(StoragePolicyException exception) { JsonXc json = new JsonXc(); Response.Status status = Response.Status.INTERNAL_SERVER_ERROR; @@ -80,7 +80,7 @@ public Response toResponse(StoragePolicyException exception) { return Response.status(status).entity(json).build(); } - public Response toResponse(LinkException exception) { + private static Response toResponse(LinkException exception) { JsonXc json = new JsonXc(); Response.Status status = Response.Status.BAD_REQUEST; json.url = exception.getLink(); diff --git a/backend/src/main/java/eu/clarin/switchboard/resources/DataResource.java b/backend/src/main/java/eu/clarin/switchboard/resources/DataResource.java index 185d23dc4..b9cf1894f 100644 --- a/backend/src/main/java/eu/clarin/switchboard/resources/DataResource.java +++ b/backend/src/main/java/eu/clarin/switchboard/resources/DataResource.java @@ -6,6 +6,7 @@ import eu.clarin.switchboard.core.FileInfo; import eu.clarin.switchboard.core.MediaLibrary; import eu.clarin.switchboard.core.xc.CommonException; +import eu.clarin.switchboard.core.xc.SwitchboardExceptionMapper; import eu.clarin.switchboard.profiler.api.ProfilingException; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; @@ -25,7 +26,7 @@ public class DataResource { private static final Logger LOGGER = LoggerFactory.getLogger(DataResource.class); - ObjectMapper mapper = new ObjectMapper(); + static ObjectMapper mapper = new ObjectMapper(); MediaLibrary mediaLibrary; public DataResource(MediaLibrary mediaLibrary) { @@ -58,6 +59,40 @@ public Response getFile(@PathParam("id") String idString) { return builder.build(); } + @GET + @Path("/{id}/info") + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + public Response getFileInfo(@Context HttpServletRequest request, @PathParam("id") String idString) + throws Exception { + UUID id; + try { + id = UUID.fromString(idString); + } catch (IllegalArgumentException xc) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + + while (true) { + Exception exception = mediaLibrary.getFileInfoAsyncError(id); + if (exception != null) { + LOGGER.debug("rethrow previous async error: {}", exception.getMessage()); + throw exception; + } + + FileInfo fi = mediaLibrary.getFileInfo(id); + if (fi == null) { + return Response.status(Response.Status.NOT_FOUND).build(); + } + + if (fi.getPath() == null) { + // async file transfer not finished, looping + continue; + } + + // async file transfer has finished + return fileInfoToResponse(request.getRequestURI(), fi); + } + } + @POST @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") @@ -76,18 +111,21 @@ public Response postFile(@Context HttpServletRequest request, return Response.status(400).entity("Please provide either a file or a url to download in the form").build(); } + return fileInfoToResponse(request.getRequestURI(), fileInfo); + } + + static Response fileInfoToResponse(String requestURI, FileInfo fileInfo) { Map ret; try { - ret = mapper.readValue(mapper.writeValueAsString(fileInfo), new TypeReference>() {}); + ret = mapper.readValue(mapper.writeValueAsString(fileInfo), new TypeReference>() { + }); } catch (JsonProcessingException xc) { LOGGER.error("json conversion exception ", xc); return Response.serverError().build(); } ret.remove("path"); - URI localLink = UriBuilder.fromPath(request.getRequestURI()) - .path(fileInfo.getId().toString()) - .build(); + URI localLink = UriBuilder.fromPath(requestURI).path(fileInfo.getId().toString()).build(); ret.put("localLink", localLink); LOGGER.debug("postFile returns: " + ret); diff --git a/backend/src/main/java/eu/clarin/switchboard/resources/IndexView.java b/backend/src/main/java/eu/clarin/switchboard/resources/IndexView.java index 2f4b93fab..3d1d0990e 100644 --- a/backend/src/main/java/eu/clarin/switchboard/resources/IndexView.java +++ b/backend/src/main/java/eu/clarin/switchboard/resources/IndexView.java @@ -1,12 +1,43 @@ package eu.clarin.switchboard.resources; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import eu.clarin.switchboard.app.SwitchboardApp; import io.dropwizard.views.View; +import java.util.UUID; + public class IndexView extends View { + static ObjectMapper mapper = new ObjectMapper(); + + static class Data { + public UUID fileInfoID; + public String errorMessage; + public boolean popup; + } + String appContextPath = SwitchboardApp.APP_CONTEXT_PATH; + String data = null; public IndexView() { super("index.mustache"); } + + public static IndexView fileInfoID(UUID id, boolean popup) throws JsonProcessingException { + IndexView view = new IndexView(); + Data data = new Data(); + data.fileInfoID = id; + data.popup = popup; + view.data = mapper.writeValueAsString(data); + return view; + } + + public static IndexView error(String errorMessage, boolean popup) throws JsonProcessingException { + IndexView view = new IndexView(); + Data data = new Data(); + data.errorMessage = errorMessage; + data.popup = popup; + view.data = mapper.writeValueAsString(data); + return view; + } } diff --git a/backend/src/main/java/eu/clarin/switchboard/resources/MainResource.java b/backend/src/main/java/eu/clarin/switchboard/resources/MainResource.java index 60f31d378..d38e99f67 100644 --- a/backend/src/main/java/eu/clarin/switchboard/resources/MainResource.java +++ b/backend/src/main/java/eu/clarin/switchboard/resources/MainResource.java @@ -1,9 +1,12 @@ package eu.clarin.switchboard.resources; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import eu.clarin.switchboard.app.FileAsset; +import eu.clarin.switchboard.core.MediaLibrary; import eu.clarin.switchboard.core.xc.CommonException; import eu.clarin.switchboard.profiler.api.ProfilingException; +import io.dropwizard.views.View; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; import org.slf4j.Logger; @@ -15,14 +18,17 @@ import javax.ws.rs.core.Request; import javax.ws.rs.core.Response; import java.io.InputStream; +import java.util.UUID; @Path("") public class MainResource { private static final Logger LOGGER = LoggerFactory.getLogger(MainResource.class); ObjectMapper mapper = new ObjectMapper(); + MediaLibrary mediaLibrary; - public MainResource() { + public MainResource(MediaLibrary mediaLibrary) { + this.mediaLibrary = mediaLibrary; } @GET @@ -36,11 +42,27 @@ public IndexView getIndex() { @Path("/") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.TEXT_HTML + ";charset=utf-8") - public IndexView postDataGetIndex( - // @Context Request request - @FormDataParam("file") InputStream inputStream, - @FormDataParam("file") final FormDataContentDisposition contentDispositionHeader, - @FormDataParam("url") String url) throws CommonException, ProfilingException { + public View postDataGetIndex(@FormDataParam("file") InputStream inputStream, + @FormDataParam("file") final FormDataContentDisposition contentDispositionHeader, + @FormDataParam("url") String url, + @FormDataParam("popup") boolean popup) + throws JsonProcessingException { + if (contentDispositionHeader != null) { + String filename = contentDispositionHeader.getFileName(); + UUID id = mediaLibrary.addMediaAsync(filename, inputStream); + return IndexView.fileInfoID(id, popup); + } else if (url != null) { + UUID id = mediaLibrary.addMediaAsync(url); + return IndexView.fileInfoID(id, popup); + } else { + return IndexView.error("Switchboard needs either a file or a url in the POST request", popup); + } + } + + @GET + @Path("/{name}") + @Produces(MediaType.TEXT_HTML + ";charset=utf-8") + public IndexView getIndex(String name) { return new IndexView(); } @@ -61,7 +83,6 @@ public Response getBundleMap(@Context Request request) { @GET @Path("/favicon.ico") - @Produces("image/x-icon") public Response getFavicon(@Context Request request) { FileAsset fileAsset = new FileAsset("webui/favicon.ico"); return fileAsset.makeResponse(request); diff --git a/backend/src/main/resources/eu/clarin/switchboard/resources/index.mustache b/backend/src/main/resources/eu/clarin/switchboard/resources/index.mustache index eb1306ac6..192213a72 100644 --- a/backend/src/main/resources/eu/clarin/switchboard/resources/index.mustache +++ b/backend/src/main/resources/eu/clarin/switchboard/resources/index.mustache @@ -13,59 +13,17 @@ + +{{#data}} + +{{/data}} +
-
- -
-
-
-
-
- About -
v2.2.1-RC3-03cecfa-SNAPSHOT
-
-
-
- Service provided by CLARIN -
-
-
-
Contact
-
- -