Skip to content

Commit

Permalink
enable POST calls and streamline popup view
Browse files Browse the repository at this point in the history
  • Loading branch information
@emanueldima @andmor-
emanueldima authored and andmor- committed Jun 4, 2020
1 parent a83a2ff commit 024472b
Show file tree
Hide file tree
Showing 18 changed files with 321 additions and 118 deletions.
2 changes: 1 addition & 1 deletion backend/src/main/java/eu/clarin/switchboard/app/SwitchboardApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public void run(RootConfig configuration, Environment environment) throws IOExce
environment.jersey().register(infoResource);
environment.jersey().register(dataResource);
environment.jersey().register(toolsResource);
environment.jersey().register(new MainResource());
environment.jersey().register(new MainResource(mediaLibrary));

environment.healthChecks().register("switchboard", new AppHealthCheck());
}
Expand Down
2 changes: 1 addition & 1 deletion backend/src/main/java/eu/clarin/switchboard/core/FileInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public FileInfo(UUID id, String filename, Path path) {
this.path = path;

this.creation = new Date().toInstant();
this.fileLength = path.toFile().length();
this.fileLength = path == null ? -1 : path.toFile().length();
}

public UUID getId() {
Expand Down
99 changes: 86 additions & 13 deletions backend/src/main/java/eu/clarin/switchboard/core/MediaLibrary.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package eu.clarin.switchboard.core;

import com.google.common.io.ByteStreams;
import eu.clarin.switchboard.app.config.DataStoreConfig;
import eu.clarin.switchboard.app.config.UrlResolverConfig;
import eu.clarin.switchboard.core.xc.CommonException;
Expand All @@ -10,7 +9,6 @@
import eu.clarin.switchboard.profiler.api.Profile;
import eu.clarin.switchboard.profiler.api.Profiler;
import eu.clarin.switchboard.profiler.api.ProfilingException;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.cache.CacheConfig;
Expand All @@ -23,6 +21,7 @@
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
Expand All @@ -39,20 +38,38 @@ public class MediaLibrary {
private final DataStore dataStore;
private final Profiler profiler;
private final StoragePolicy storagePolicy;
private final UrlResolverConfig urlResolverConfig;
private final CloseableHttpClient cachingClient;
private final ExecutorService executorService;

Map<UUID, FileInfo> fileInfoMap = Collections.synchronizedMap(new HashMap<>());

public static class FileError {
Instant creation;
Exception exception;

public FileError(Exception exception) {
this.creation = Instant.now();
this.exception = exception;
}

public Instant getCreation() {
return creation;
}

public Exception getException() {
return exception;
}
}
Map<UUID, FileError> fileInfoAsyncErrorMap = Collections.synchronizedMap(new HashMap<>());

public MediaLibrary(DataStore dataStore, Profiler profiler, StoragePolicy storagePolicy,
UrlResolverConfig urlResolver, DataStoreConfig dataStoreConfig) {
UrlResolverConfig urlResolverConfig, DataStoreConfig dataStoreConfig) {
this.dataStore = dataStore;
this.profiler = profiler;
this.storagePolicy = storagePolicy;
this.urlResolverConfig = urlResolver;

CacheConfig cacheConfig = CacheConfig.custom()
.setMaxCacheEntries(urlResolver.getMaxHttpCacheEntries())
.setMaxCacheEntries(urlResolverConfig.getMaxHttpCacheEntries())
.setMaxObjectSize(dataStoreConfig.getMaxSize())
.build();
RequestConfig requestConfig = RequestConfig.custom()
Expand All @@ -66,12 +83,64 @@ public MediaLibrary(DataStore dataStore, Profiler profiler, StoragePolicy storag
.setDefaultRequestConfig(requestConfig)
.build();

executorService = Executors.newCachedThreadPool();

ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
Duration cleanup = storagePolicy.getCleanupPeriod();
executor.scheduleAtFixedRate(this::periodicCleanup, cleanup.getSeconds(), cleanup.getSeconds(), TimeUnit.SECONDS);
}

public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException, ProfilingException {
UUID id = UUID.randomUUID();
return addMedia(id, originalUrlOrDoiOrHandle);
}

public UUID addMediaAsync(String originalUrlOrDoiOrHandle) {
UUID id = UUID.randomUUID();
fileInfoMap.put(id, new FileInfo(id, null, null));
executorService.submit(() -> {
try {
addMedia(id, originalUrlOrDoiOrHandle);
} catch (Exception exception) {
LOGGER.debug("async error: {}", exception.getMessage());
fileInfoAsyncErrorMap.put(id, new FileError(exception));
fileInfoMap.remove(id);
}
});
return id;
}

public FileInfo addMedia(String filename, InputStream inputStream) throws
StoragePolicyException, StorageException, ProfilingException {
UUID id = UUID.randomUUID();
return addMedia(id, filename, inputStream);
}

public UUID addMediaAsync(String filename, InputStream inputStream) {
UUID id = UUID.randomUUID();
fileInfoMap.put(id, new FileInfo(id, null, null));
executorService.submit(() -> {
try {
addMedia(id, filename, inputStream);
} catch (Exception exception) {
LOGGER.debug("async error: {}", exception.getMessage());
fileInfoAsyncErrorMap.put(id, new FileError(exception));
fileInfoMap.remove(id);
}
});
return id;
}

public FileInfo getFileInfo(UUID id) {
return fileInfoMap.get(id);
}

public Exception getFileInfoAsyncError(UUID id) {
FileError fileError = fileInfoAsyncErrorMap.get(id);
return fileError == null ? null : fileError.getException();
}

private FileInfo addMedia(UUID id, String originalUrlOrDoiOrHandle) throws CommonException, ProfilingException {
LinkMetadata.LinkInfo linkInfo = LinkMetadata.getLinkData(cachingClient, originalUrlOrDoiOrHandle);
try {
if (linkInfo.response.getEntity().getContentLength() > storagePolicy.getMaxAllowedDataSize()) {
Expand All @@ -80,7 +149,7 @@ public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException
DefaultStoragePolicy.humanSize(storagePolicy.getMaxAllowedDataSize()) + ".",
StoragePolicyException.Kind.TOO_BIG);
}
FileInfo fileInfo = addMedia(linkInfo.filename, linkInfo.response.getEntity().getContent());
FileInfo fileInfo = addMedia(id, linkInfo.filename, linkInfo.response.getEntity().getContent());
fileInfo.setLinksInfo(originalUrlOrDoiOrHandle, linkInfo.downloadLink, linkInfo.redirects);
return fileInfo;
} catch (IOException xc) {
Expand All @@ -94,9 +163,8 @@ public FileInfo addMedia(String originalUrlOrDoiOrHandle) throws CommonException
}
}

public FileInfo addMedia(String filename, InputStream inputStream) throws
private FileInfo addMedia(UUID id, String filename, InputStream inputStream) throws
StoragePolicyException, StorageException, ProfilingException {
UUID id = UUID.randomUUID();
Path path;
try {
path = dataStore.save(id, filename, inputStream);
Expand Down Expand Up @@ -138,10 +206,6 @@ public FileInfo addMedia(String filename, InputStream inputStream) throws
return fileInfo;
}

public FileInfo getFileInfo(UUID id) {
return fileInfoMap.get(id);
}

private void periodicCleanup() {
// this runs on its own thread
LOGGER.info("start periodic cleanup now");
Expand All @@ -154,5 +218,14 @@ private void periodicCleanup() {
iterator.remove();
}
}

LOGGER.info("start periodic error cleanup now");
for (Iterator<FileError> iterator = fileInfoAsyncErrorMap.values().iterator(); iterator.hasNext(); ) {
FileError fe = iterator.next();
Duration lifetime = Duration.between(fe.getCreation(), Instant.now());
if (lifetime.compareTo(storagePolicy.getMaxAllowedLifetime()) > 0) {
iterator.remove();
}
}
}
}
4 changes: 2 additions & 2 deletions backend/src/main/java/eu/clarin/switchboard/core/xc/SwitchboardExceptionMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public Response toResponse(Exception exception) {
return Response.status(status).entity(json).build();
}

public Response toResponse(StoragePolicyException exception) {
private static Response toResponse(StoragePolicyException exception) {
JsonXc json = new JsonXc();
Response.Status status = Response.Status.INTERNAL_SERVER_ERROR;

Expand All @@ -80,7 +80,7 @@ public Response toResponse(StoragePolicyException exception) {
return Response.status(status).entity(json).build();
}

public Response toResponse(LinkException exception) {
private static Response toResponse(LinkException exception) {
JsonXc json = new JsonXc();
Response.Status status = Response.Status.BAD_REQUEST;
json.url = exception.getLink();
Expand Down
48 changes: 43 additions & 5 deletions backend/src/main/java/eu/clarin/switchboard/resources/DataResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import eu.clarin.switchboard.core.FileInfo;
import eu.clarin.switchboard.core.MediaLibrary;
import eu.clarin.switchboard.core.xc.CommonException;
import eu.clarin.switchboard.core.xc.SwitchboardExceptionMapper;
import eu.clarin.switchboard.profiler.api.ProfilingException;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataParam;
Expand All @@ -25,7 +26,7 @@
public class DataResource {
private static final Logger LOGGER = LoggerFactory.getLogger(DataResource.class);

ObjectMapper mapper = new ObjectMapper();
static ObjectMapper mapper = new ObjectMapper();
MediaLibrary mediaLibrary;

public DataResource(MediaLibrary mediaLibrary) {
Expand Down Expand Up @@ -58,6 +59,40 @@ public Response getFile(@PathParam("id") String idString) {
return builder.build();
}

@GET
@Path("/{id}/info")
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
public Response getFileInfo(@Context HttpServletRequest request, @PathParam("id") String idString)
throws Exception {
UUID id;
try {
id = UUID.fromString(idString);
} catch (IllegalArgumentException xc) {
return Response.status(Response.Status.NOT_FOUND).build();
}

while (true) {
Exception exception = mediaLibrary.getFileInfoAsyncError(id);
if (exception != null) {
LOGGER.debug("rethrow previous async error: {}", exception.getMessage());
throw exception;
}

FileInfo fi = mediaLibrary.getFileInfo(id);
if (fi == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}

if (fi.getPath() == null) {
// async file transfer not finished, looping
continue;
}

// async file transfer has finished
return fileInfoToResponse(request.getRequestURI(), fi);
}
}

@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
Expand All @@ -76,18 +111,21 @@ public Response postFile(@Context HttpServletRequest request,
return Response.status(400).entity("Please provide either a file or a url to download in the form").build();
}

return fileInfoToResponse(request.getRequestURI(), fileInfo);
}

static Response fileInfoToResponse(String requestURI, FileInfo fileInfo) {
Map<String, Object> ret;
try {
ret = mapper.readValue(mapper.writeValueAsString(fileInfo), new TypeReference<Map<String, Object>>() {});
ret = mapper.readValue(mapper.writeValueAsString(fileInfo), new TypeReference<Map<String, Object>>() {
});
} catch (JsonProcessingException xc) {
LOGGER.error("json conversion exception ", xc);
return Response.serverError().build();
}
ret.remove("path");

URI localLink = UriBuilder.fromPath(request.getRequestURI())
.path(fileInfo.getId().toString())
.build();
URI localLink = UriBuilder.fromPath(requestURI).path(fileInfo.getId().toString()).build();
ret.put("localLink", localLink);

LOGGER.debug("postFile returns: " + ret);
Expand Down
31 changes: 31 additions & 0 deletions backend/src/main/java/eu/clarin/switchboard/resources/IndexView.java
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,43 @@
package eu.clarin.switchboard.resources;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.clarin.switchboard.app.SwitchboardApp;
import io.dropwizard.views.View;

import java.util.UUID;

public class IndexView extends View {
static ObjectMapper mapper = new ObjectMapper();

static class Data {
public UUID fileInfoID;
public String errorMessage;
public boolean popup;
}

String appContextPath = SwitchboardApp.APP_CONTEXT_PATH;
String data = null;

public IndexView() {
super("index.mustache");
}

public static IndexView fileInfoID(UUID id, boolean popup) throws JsonProcessingException {
IndexView view = new IndexView();
Data data = new Data();
data.fileInfoID = id;
data.popup = popup;
view.data = mapper.writeValueAsString(data);
return view;
}

public static IndexView error(String errorMessage, boolean popup) throws JsonProcessingException {
IndexView view = new IndexView();
Data data = new Data();
data.errorMessage = errorMessage;
data.popup = popup;
view.data = mapper.writeValueAsString(data);
return view;
}
}
Loading

0 comments on commit 024472b

Please sign in to comment.