Merge pull request #137 from sbesson/omero-zarr-pixel-buffer_0.3.0

Zarr pixel buffer performance improvements

build.gradle

@@ -8,11 +8,12 @@ version = '0.9.0-SNAPSHOT'
 
 mainClassName = 'com.glencoesoftware.omero.ms.image.region.OmeroVertxLauncher'
 
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
+sourceCompatibility = 1.11
+targetCompatibility = 1.11
 
 repositories {
     mavenCentral()
+    mavenLocal()
     maven {
         name 'Unidata'
         url 'https://artifacts.unidata.ucar.edu/content/repositories/unidata-releases'
@@ -29,16 +30,9 @@ repositories {
     maven {
         url 'https://repo.glencoesoftware.com/repository/bioformats2raw2ometiff/'
     }
-    maven {
-        url 'https://repo.glencoesoftware.com/repository/jzarr-snapshots'
-    }
     maven {
         url 'https://artifacts.glencoesoftware.com/artifactory/scijava-thirdparty'
     }
-    maven {
-        name 'Brockmann-Consult Public Maven Repository'
-        url 'https://nexus.senbox.net/nexus/content/groups/public/'
-    }
 }
 
 configurations.all {
@@ -68,7 +62,8 @@ dependencies {
     implementation 'io.zipkin.reporter2:zipkin-sender-okhttp3:2.10.0'
     implementation 'ch.qos.logback:logback-classic:1.3.14'
     implementation 'org.slf4j:log4j-over-slf4j:1.7.32'
-    implementation 'com.glencoesoftware.omero:omero-ms-core:0.7.0'
+    implementation 'com.glencoesoftware.omero:omero-zarr-pixel-buffer:0.3.0-SNAPSHOT'
+    implementation 'com.glencoesoftware.omero:omero-ms-core:0.8.0-SNAPSHOT'
     implementation 'io.vertx:vertx-web:3.8.1'
     implementation 'io.vertx:vertx-config:3.8.1'
     implementation 'io.vertx:vertx-config-yaml:3.8.1'
@@ -81,14 +76,15 @@ dependencies {
     implementation 'com.univocity:univocity-parsers:2.8.3'
     implementation 'com.zeroc:icegrid:3.6.4'
     implementation 'com.google.code.gson:gson:2.8.6'
-    implementation 'com.bc.zarr:jzarr:0.3.4'
+    implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8'
+    implementation 'dev.zarr:jzarr:0.4.2'
     implementation 'org.lasersonlab:s3fs:2.2.3'
-    implementation 'com.amazonaws:aws-java-sdk-s3:1.11.1034'
+    implementation 'com.amazonaws:aws-java-sdk-s3:1.12.659'
 
     testImplementation("junit:junit:4.12")
     testImplementation 'org.mockito:mockito-core:2.28.2'
     testImplementation 'com.glencoesoftware:bioformats2raw:0.9.1'
-
+    testImplementation 'info.picocli:picocli:4.7.5'
 }
 
 jar {

src/dist/conf/config.yaml

@@ -24,12 +24,14 @@ omero.server:
     omero.db.pass: "omero"
     # OMERO_HOME/lib/scripts
     omero.script_repo_root: "/opt/omero/OMERO.current/lib/scripts"
+    omero.pixeldata.memoizer.dir: "/OMERO/BioFormatsCache"
     omero.pixeldata.max_tile_length: "2048"
     omero.pixeldata.max_plane_width: "3192"
     omero.pixeldata.max_plane_height: "3192"
     # The following should be set to match your OMERO.web settings
     # See https://docs.openmicroscopy.org/omero/5.6.3/sysadmins/config.html#omero-client-viewer-interpolate-pixels
     # and https://docs.openmicroscopy.org/omero/5.6.3/sysadmins/config.html#omero-client-viewer-initial-zoom-level
+    omero.pixeldata.zarr_cache_size: "500"
     omero.client.viewer.initial_zoom_level: "0"
     omero.client.viewer.interpolate_pixels: "true"
 # OMERO.web configuration

src/dist/memo_regenerator.sql

@@ -8,9 +8,11 @@ COPY (SELECT * FROM (
            pixels.sizeZ,
            pixels.sizeC,
            pixels.sizeT,
+           format.value,
            rank() OVER (PARTITION BY fileset.id ORDER BY image.id) AS rank
         FROM fileset
             JOIN image ON fileset.id = image.fileset
             JOIN pixels ON image.id = pixels.image
             JOIN pixelstype ON pixels.pixelstype = pixelstype.id
+            JOIN format ON image.format = format.id
 ) AS rank WHERE rank.rank = 1) TO STDOUT CSV;

src/main/java/com/glencoesoftware/omero/ms/image/region/HistogramRequestHandler.java

@@ -25,7 +25,6 @@
 
 import org.slf4j.LoggerFactory;
 
-
 import brave.ScopedSpan;
 import brave.Tracer;
 import brave.Tracing;

src/main/java/com/glencoesoftware/omero/ms/image/region/ImageDataRequestHandler.java

@@ -627,6 +627,7 @@ protected Image queryImageData(IQueryPrx iQuery, Long imageId)
                     .findByQuery("select i from Image as i"
                             + " left outer join fetch i.details.externalInfo "
                             + " join fetch i.pixels as p"
+                            + " join fetch i.format"
                             + " left outer JOIN FETCH i.datasetLinks as links "
                             + " left outer join fetch links.parent as dataset "
                             + " left outer join fetch dataset.projectLinks as plinks "

src/main/java/com/glencoesoftware/omero/ms/image/region/ImageRegionMicroserviceVerticle.java

@@ -171,8 +171,7 @@ public void deploy(JsonObject config, Promise<Void> prom) {
         context = new ClassPathXmlApplicationContext(
                 "classpath:ome/config.xml",
                 "classpath:ome/services/datalayer.xml",
-                "classpath*:beanRefContext.xml",
-                "classpath*:service-ms.core.PixelsService.xml");
+                "classpath*:beanRefContext.xml");
         preferences =
                 (PreferenceContext) this.context.getBean("preferenceContext");
         JsonObject httpTracingConfig =

src/main/java/com/glencoesoftware/omero/ms/image/region/ImageRegionRequestHandler.java

@@ -150,6 +150,7 @@ protected Map<Long, Pixels> retrievePixDescription(
                     iQuery.findAllByQuery(
                         "select p from Pixels as p "
                         + "join fetch p.image as i "
+                        + "join fetch i.format "
                         + "left outer join fetch i.details.externalInfo "
                         + "left outer join fetch i.wellSamples as ws "
                         + "left outer join fetch ws.well as w "

src/main/java/com/glencoesoftware/omero/ms/image/region/MemoRegenerator.java

@@ -36,9 +36,11 @@
 
 import loci.formats.FormatTools;
 import ome.conditions.MissingPyramidException;
+import ome.conditions.LockTimeout;
 import ome.io.nio.PixelBuffer;
 import ome.io.nio.PixelsService;
 import ome.model.core.Image;
+import ome.model.enums.Format;
 import ome.model.core.Pixels;
 import ome.model.enums.PixelsType;
 import picocli.CommandLine;
@@ -158,8 +160,7 @@ private void init(Map<String, Object> config) {
         context = new ClassPathXmlApplicationContext(
                 "classpath:ome/config.xml",
                 "classpath:ome/services/datalayer.xml",
-                "classpath*:beanRefContext.xml",
-                "classpath*:service-ms.core.PixelsService.xml");
+                "classpath*:beanRefContext.xml");
         pixelsService = (PixelsService) context.getBean("/OMERO/Pixels");
         if (mode.cacheDir != null) {
             pixelsService.setMemoizerDirectoryLocal(mode.cacheDir.toString());
@@ -197,7 +198,7 @@ private void regen() {
                 }
                 long elapsedTime = System.nanoTime() - startTime;
                 System.out.printf("%d/%d - ok: %d %.3f%n", i, total, imageId, (float) elapsedTime/1000000);
-            } catch (MissingPyramidException e) {
+            } catch (MissingPyramidException | LockTimeout e) {
                 long elapsedTime = System.nanoTime() - startTime;
                 System.out.printf("%d/%d - skip: %d %.3f%n", i, total, imageId, (float) elapsedTime/1000000);
             } catch (Exception e) {
@@ -224,10 +225,11 @@ private Pixels pixelsFromRow(Object[] row) {
         Integer sizeZ = (Integer) row[6];
         Integer sizeC = (Integer) row[7];
         Integer sizeT = (Integer) row[8];
+        String format = (String) row[9];
 
         Image image = new Image(imageId, true);
         image.setSeries(series);
-
+        image.setFormat(new Format(format));
         Pixels pixels = new Pixels(pixelsId, true);
         pixels.setImage(image);
         PixelsType pt = new PixelsType(pixelsType);