Skip to content

Commit

Permalink
Merge branch 'dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
@bastiao
bastiao committed Aug 28, 2024
2 parents 0a14f0f + ad82398 commit 0764818
Show file tree
Hide file tree
Showing 13 changed files with 441 additions and 92 deletions.
11 changes: 11 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,17 @@
# CHANGELOG
This document intends to keep track of the changes performed on the various releases of Dicoogle.

## 3.4

### 3.4.0 (2024-08-27)

* New: Introduced bulk unindexing functionality for IndexerInterface. (#598)
* Enhancement: Added a kill switch for dead plugin sets in the core. (#686)
* Enhancement: Improved logging around indexing tasks. (#691)
* Enhancement: Added robustness to DIMGeneric against BodyPartThickness. (#694)
* Enhancement: Improved StorageInterface by adding a `get` method. (#660)
* Enhancement: Undeprecated StorageInterface#handles in the SDK. (#695)

## 3.3

### 3.3.6 (2024-04-24)
Expand Down
2 changes: 1 addition & 1 deletion dicoogle/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<parent>
<groupId>pt.ua.ieeta</groupId>
<artifactId>dicoogle-all</artifactId>
<version>3.3.6</version>
<version>3.4.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down
11 changes: 2 additions & 9 deletions dicoogle/src/main/java/pt/ua/dicoogle/plugins/DefaultFileStoragePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,6 @@ public String getScheme() {
return defaultScheme;
}

@Override
public boolean handles(URI location) {
if (location.getScheme() == null)
return true;
return location.getScheme().equals(defaultScheme);
}

private Iterator<StorageInputStream> createIterator(URI location) {
if (!handles(location)) {
logger.error("Cannot Handle: " + location.toString());
Expand Down Expand Up @@ -122,7 +115,7 @@ public URI store(DicomInputStream inputStream, Object... args) throws IOExceptio

@Override
public void remove(URI location) {
if (!location.getScheme().equals(defaultScheme)) {
if (!handles(location)) {
return;
}

Expand All @@ -135,7 +128,7 @@ public void remove(URI location) {

@Override
public Stream<URI> list(URI location) throws IOException {
if (!location.getScheme().equals(defaultScheme)) {
if (!handles(location)) {
return Stream.empty();
}

Expand Down
80 changes: 56 additions & 24 deletions dicoogle/src/main/java/pt/ua/dicoogle/plugins/PluginController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import pt.ua.dicoogle.plugins.webui.WebUIPluginManager;
import pt.ua.dicoogle.sdk.*;
import pt.ua.dicoogle.sdk.datastructs.Report;
import pt.ua.dicoogle.sdk.datastructs.UnindexReport;
import pt.ua.dicoogle.sdk.datastructs.SearchResult;
import pt.ua.dicoogle.sdk.datastructs.dim.DimLevel;
import pt.ua.dicoogle.sdk.settings.ConfigurationHolder;
Expand All @@ -45,6 +46,7 @@
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.zip.ZipFile;

Expand Down Expand Up @@ -89,6 +91,9 @@ public synchronized static PluginController getInstance() {
private TaskManager taskManagerQueries =
new TaskManager(Integer.parseInt(System.getProperty("dicoogle.taskManager.nQueryThreads", "4")));

/** Whether to shut down Dicoogle when a plugin is marked as dead */
private static boolean DEAD_PLUGIN_KILL_SWITCH =
System.getProperty("dicoogle.deadPluginKillSwitch", "false").equalsIgnoreCase("true");

public PluginController(File pathToPluginDirectory) {
logger.info("Creating PluginController Instance");
Expand Down Expand Up @@ -172,9 +177,14 @@ private void configurePlugins() {
logger.warn("Plugin set name cannot be retrieved: {}", ex2.getMessage());
name = "UNKNOWN";
}
logger.error("Unexpected error while loading plugin set {}. Plugin set marked as dead.", name, e);
this.deadPluginSets.add(new DeadPlugin(name, e));
it.remove();
if (DEAD_PLUGIN_KILL_SWITCH) {
logger.error("Unexpected error while loading plugin set {}. Dicoogle will shut down.", name, e);
System.exit(-4);
} else {
logger.error("Unexpected error while loading plugin set {}. Plugin set marked as dead.", name, e);
this.deadPluginSets.add(new DeadPlugin(name, e));
it.remove();
}
}
}
logger.debug("Settings pushed to plugins");
Expand Down Expand Up @@ -446,15 +456,13 @@ public List<String> getQueryProvidersName(boolean enabled) {
for (QueryInterface p : plugins) {
names.add(p.getName());
}
// logger.info("Query Providers: "+Arrays.toString(names.toArray()) );
return names;
}

public QueryInterface getQueryProviderByName(String name, boolean onlyEnabled) {
Collection<QueryInterface> plugins = getQueryPlugins(onlyEnabled);
for (QueryInterface p : plugins) {
if (p.getName().equalsIgnoreCase(name)) {
// logger.info("Retrived Query Provider: "+name);
return p;
}
}
Expand Down Expand Up @@ -486,7 +494,6 @@ public IndexerInterface getIndexerByName(String name, boolean onlyEnabled) {
Collection<IndexerInterface> plugins = getIndexingPlugins(onlyEnabled);
for (IndexerInterface p : plugins) {
if (p.getName().equalsIgnoreCase(name)) {
// logger.info("Retrived Query Provider: "+name);
return p;
}
}
Expand All @@ -499,7 +506,6 @@ public JettyPluginInterface getServletByName(String name, boolean onlyEnabled) {
Collection<JettyPluginInterface> plugins = getServletPlugins(onlyEnabled);
for (JettyPluginInterface p : plugins) {
if (p.getName().equalsIgnoreCase(name)) {
// logger.info("Retrived Query Provider: "+name);
return p;
}
}
Expand All @@ -511,7 +517,6 @@ public StorageInterface getStorageByName(String name, boolean onlyEnabled) {
Collection<StorageInterface> plugins = getStoragePlugins(onlyEnabled);
for (StorageInterface p : plugins) {
if (p.getName().equalsIgnoreCase(name)) {
// logger.info("Retrived Query Provider: "+name);
return p;
}
}
Expand All @@ -527,7 +532,6 @@ public JointQueryTask queryAll(JointQueryTask holder, final String query, final

public JointQueryTask queryAll(JointQueryTask holder, final String query, final DimLevel level,
final Object... parameters) {
// logger.info("Querying all providers");
List<String> providers = this.getQueryProvidersName(true);
return query(holder, providers, query, level, parameters);
}
Expand All @@ -544,7 +548,6 @@ public Task<Iterable<SearchResult>> query(String querySource, final String query
final Object... parameters) {
Task<Iterable<SearchResult>> t = getTaskForQueryDim(querySource, query, level, parameters);
taskManagerQueries.dispatch(t);
// logger.info("Fired Query Task: "+querySource +" QueryString:"+query);

return t;// returns the handler to obtain the computation results
}
Expand All @@ -565,7 +568,6 @@ public JointQueryTask query(JointQueryTask holder, List<String> querySources, fi
for (Task<?> t : tasks)
taskManagerQueries.dispatch(t);

// logger.info("Fired Query Tasks: "+Arrays.toString(querySources.toArray()) +" QueryString:"+query);
return holder;// returns the handler to obtain the computation results
}

Expand All @@ -585,7 +587,6 @@ public JointQueryTask query(JointQueryTask holder, List<String> querySources, fi
for (Task<?> t : tasks)
taskManagerQueries.dispatch(t);

// logger.info("Fired Query Tasks: "+Arrays.toString(querySources.toArray()) +" QueryString:"+query);
return holder;// returns the handler to obtain the computation results
}

Expand All @@ -610,7 +611,6 @@ public Iterable<SearchResult> call() throws Exception {

}
});
// logger.info("Prepared Query Task: QueryString");
return queryTask;
}

Expand Down Expand Up @@ -646,7 +646,6 @@ public Iterable<SearchResult> call() {

}
});
// logger.info("Prepared Query Task: QueryString");
return queryTask;
}

Expand Down Expand Up @@ -679,11 +678,8 @@ public List<Task<Report>> index(URI path) {
continue;
final String taskUniqueID = UUID.randomUUID().toString();
task.setName(String.format("[%s]index %s", indexer.getName(), path));
task.onCompletion(new Runnable() {
@Override
public void run() {
logger.info("Task [{}] complete: {} is indexed", taskUniqueID, pathF);
}
task.onCompletion(() -> {
logger.info("Task [{}] complete on {}", taskUniqueID, pathF);
});

taskManager.dispatch(task);
Expand All @@ -693,7 +689,7 @@ public void run() {
logger.warn("Indexer {} failed unexpectedly", indexer.getName(), ex);
}
}
logger.info("Finished firing all indexing plugins for {}", path);
logger.debug("Finished firing all indexing plugins for {}", path);

return rettasks;
}
Expand All @@ -720,14 +716,14 @@ public List<Task<Report>> index(String pluginName, URI path) {

@Override
public void run() {
logger.info("Task [{}] complete: {} is indexed", taskUniqueID, pathF);
logger.info("Task [{}] complete on {}", taskUniqueID, pathF);
}
});

taskManager.dispatch(task);

rettasks.add(task);
logger.info("Fired indexer {} for URI {}", pluginName, path.toString());
logger.debug("Fired indexer {} for URI {}", pluginName, path.toString());
RunningIndexTasks.getInstance().addTask(task);
}
} catch (RuntimeException ex) {
Expand Down Expand Up @@ -761,7 +757,43 @@ public void unindex(URI path, Collection<String> indexProviders) {
}
}

/** Issue an unindexation procedure to the given indexers.
/** Issue the removal of indexed entries in bulk.
*
* @param indexProvider the name of the indexer
* @param items a collections of item identifiers to unindex
* @param progressCallback an optional function (can be `null`),
* called for every batch of items successfully unindexed
* to indicate early progress
* and inform consumers that
* it is safe to remove or exclude the unindexed item
* @return an asynchronous task object returning
* a report containing which files were not unindexed,
* and whether some of them were not found in the database
* @throws IOException
*/
public Task<UnindexReport> unindex(String indexProvider, Collection<URI> items, Consumer<Collection<URI>> progressCallback) throws IOException {
logger.info("Starting unindexing procedure for {} items", items.size());

IndexerInterface indexer = null;
if (indexProvider != null) {
indexer = this.getIndexerByName(indexProvider, true);
}
if (indexer == null) {
indexer = this.getIndexingPlugins(true).iterator().next();
}
Task<UnindexReport> task = indexer.unindex(items, progressCallback);
if (task != null) {
final String taskUniqueID = UUID.randomUUID().toString();
task.setName(String.format("[%s]unindex", indexer.getName()));
task.onCompletion(() -> {
logger.info("Unindexing task [{}] complete", taskUniqueID);
});
taskManager.dispatch(task);
}
return task;
}

/** Issue an unindexing procedure to the given indexers.
*
* @param path the URI of the directory or file to unindex
* @param indexers a collection of providers
Expand All @@ -782,7 +814,7 @@ public void remove(URI uri) {
}

public void doRemove(URI uri, StorageInterface si) {
if (si.handles(uri)) {
if (Objects.equals(uri.getScheme(), si.getScheme())) {
si.remove(uri);
} else {
logger.warn("Storage Plugin does not handle URI: {},{}", uri, si);
Expand Down
7 changes: 2 additions & 5 deletions dicoogle/src/main/java/pt/ua/dicoogle/server/web/dicom/Information.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,20 +113,17 @@ public void onReceive(Task<Iterable<SearchResult>> e) {
for (SearchResult r : itResults) {
if (uri == null)
uri = r.getURI();
System.out.println("URI: " + uri.toString());
}

if (uri != null) {
StorageInterface str = PluginController.getInstance().getStorageForSchema(uri);
if (str != null) {
Iterable<StorageInputStream> stream = str.at(uri);
for (StorageInputStream r : stream) {
StorageInputStream r = str.get(uri);
if (r != null) {
ret = r;

stopAllTaks();

latch.countDown();

return;
}
}
Expand Down
29 changes: 18 additions & 11 deletions dicoogle/src/main/java/pt/ua/dicoogle/server/web/servlets/ImageServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,13 +127,13 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response)
// get the image file by the URI
URI imgUri = new URI(uri);
StorageInterface storageInt = PluginController.getInstance().getStorageForSchema(imgUri);
Iterator<StorageInputStream> storages = storageInt.at(imgUri).iterator();
StorageInputStream storageItem = storageInt.get(imgUri);
// take the first valid storage
if (!storages.hasNext()) {
if (storageItem == null) {
ResponseUtil.sendError(response, 404, "No image file for supplied URI");
return;
}
imgFile = storages.next();
imgFile = storageItem;

} catch (URISyntaxException ex) {
ResponseUtil.sendError(response, 400, "Bad URI syntax");
Expand Down Expand Up @@ -220,7 +220,6 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S

private static StorageInputStream getFileFromSOPInstanceUID(String sopInstanceUID, List<String> providers)
throws IOException {
// TODO use only DIM sources?
JointQueryTask qt = new JointQueryTask() {
@Override
public void onCompletion() {}
Expand All @@ -229,24 +228,32 @@ public void onCompletion() {}
public void onReceive(Task<Iterable<SearchResult>> e) {}
};
try {
PluginController pc = PluginController.getInstance();
if (providers == null) {
providers = PluginController.getInstance().getQueryProvidersName(true);
// use only DIM sources
providers = ServerSettingsManager.getSettings().getArchiveSettings().getDIMProviders();
// exclude unknown query providers
providers.removeIf(pName -> pc.getQueryProviderByName(pName, true) == null);
if (providers.isEmpty()) {
// fallback to all query providers
providers = pc.getQueryProvidersName(true);
}
}
Iterator<SearchResult> it = PluginController.getInstance()
.query(qt, providers, "SOPInstanceUID:" + sopInstanceUID).get().iterator();
Iterator<SearchResult> it = pc
.query(qt, providers, "SOPInstanceUID:\"" + sopInstanceUID + '"').get().iterator();
if (!it.hasNext()) {
throw new IOException("No such image of SOPInstanceUID " + sopInstanceUID);
}
SearchResult res = it.next();
StorageInterface storage = PluginController.getInstance().getStorageForSchema(res.getURI());
StorageInterface storage = pc.getStorageForSchema(res.getURI());
if (storage == null) {
throw new IOException("Unsupported file scheme");
}
Iterator<StorageInputStream> store = storage.at(res.getURI()).iterator();
if (!store.hasNext()) {
StorageInputStream item = storage.get(res.getURI());
if (item == null) {
throw new IOException("No storage item found");
}
return store.next();
return item;
} catch (InterruptedException | ExecutionException ex) {
throw new IOException(ex);
}
Expand Down
Loading

0 comments on commit 0764818

Please sign in to comment.