driver: don't flush empty files or files without checksum

dCache · Jun 13, 2023 · 5bfdfa6 · 5bfdfa6
1 parent 4b0b7c8
commit 5bfdfa6
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 1 deletion.
diff --git a/src/main/java/org/dcache/nearline/cta/CtaNearlineStorage.java b/src/main/java/org/dcache/nearline/cta/CtaNearlineStorage.java
@@ -39,12 +39,15 @@
 import ch.cern.cta.rpc.RetrieveResponse;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import org.dcache.namespace.FileAttribute;
 import org.dcache.nearline.cta.xrootd.DataMover;
 import org.dcache.pool.nearline.spi.FlushRequest;
 import org.dcache.pool.nearline.spi.NearlineStorage;
 import org.dcache.pool.nearline.spi.RemoveRequest;
 import org.dcache.pool.nearline.spi.StageRequest;
 import org.dcache.util.Checksum;
+import org.dcache.util.ChecksumType;
+import org.dcache.vehicles.FileAttributes;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -190,6 +193,23 @@ public void flush(Iterable<FlushRequest> requests) {
 
             try {
                 fr.activate().get();
+
+                // don't flush zero-byte files
+                if (fr.getFileAttributes().getSize() == 0) {
+                    LOGGER.info("Fake flush of zero-byte file {}", fr.getFileAttributes().getPnfsId());
+                    fr.completed(Set.of(createZeroFileUri(fr.getFileAttributes())));
+                    continue;
+                }
+
+                // CTA can't flush files without checksum, thus, even don't try
+                if (!fr.getFileAttributes().isDefined(FileAttribute.CHECKSUM) ||
+                        fr.getFileAttributes().getChecksums().isEmpty()) {
+                    LOGGER.warn("Can't flush file {} - no checksum", fr.getFileAttributes().getPnfsId());
+                    fr.failed(CacheException.NO_ATTRIBUTE, "Can't flush files without checksum");
+                    continue;
+                }
+
+
                 final AtomicLong id = new AtomicLong();
 
                 var createRequest = ctaRequestFactory.valueOf(fr.getFileAttributes());
@@ -336,7 +356,15 @@ public void completed(Set<Checksum> checksums) {
             try {
                 r.activate().get();
                 r.allocate().get();
-            } catch (ExecutionException | InterruptedException e) {
+
+                // no need to stage an empty files
+                if (sr.getFileAttributes().getSize() == 0) {
+                    new File(sr.getReplicaUri().getPath()).createNewFile();
+                    sr.completed(Set.of(new Checksum(ChecksumType.ADLER32.createMessageDigest())));
+                    continue;
+                }
+
+            } catch (IOException | ExecutionException | InterruptedException e) {
                 Throwable t = Throwables.getRootCause(e);
                 LOGGER.error("Failed to activate/allocate space for retrieve request: {}",
                       t.getMessage());
@@ -583,4 +611,8 @@ private CacheException asCacheException(Throwable e) {
 
         return new CacheException(CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
     }
+
+    private URI createZeroFileUri(FileAttributes attrs) {
+        return URI.create(type + "://" + name + "/" + attrs.getPnfsId() + "?archiveid=*");
+    }
 }
diff --git a/src/test/java/org/dcache/nearline/cta/CtaNearlineStorageTest.java b/src/test/java/org/dcache/nearline/cta/CtaNearlineStorageTest.java
@@ -34,9 +34,12 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import org.dcache.namespace.FileAttribute;
 import org.dcache.pool.nearline.spi.FlushRequest;
 import org.dcache.pool.nearline.spi.RemoveRequest;
 import org.dcache.pool.nearline.spi.StageRequest;
+import org.dcache.util.Checksum;
+import org.dcache.util.ChecksumType;
 import org.dcache.vehicles.FileAttributes;
 import org.junit.After;
 import org.junit.Before;
@@ -310,6 +313,36 @@ public void testFlushRequestFailOnRpcError() {
         verify(request).failed(any());
     }
 
+    @Test
+    public void testFlushOfEmptyFile() {
+
+        var request = mockedFlushRequest();
+        request.getFileAttributes().setSize(0L);
+
+        driver = new CtaNearlineStorage("foo", "bar");
+        driver.configure(drvConfig);
+        driver.start();
+
+        driver.flush(Set.of(request));
+        // TODO: check for correct URI
+        verify(request).completed(any());
+    }
+
+
+    @Test
+    public void testFlushWithoutChecksum() {
+
+        var request = mockedFlushRequest();
+        request.getFileAttributes().undefine(FileAttribute.CHECKSUM);
+
+        driver = new CtaNearlineStorage("foo", "bar");
+        driver.configure(drvConfig);
+        driver.start();
+
+        driver.flush(Set.of(request));
+        verify(request).failed(anyInt(), any());
+    }
+
     @Test
     public void testSuccessOnRemove() {
 
@@ -662,6 +695,7 @@ private FlushRequest mockedFlushRequest() {
               .hsm("z")
               .creationTime(System.currentTimeMillis())
               .pnfsId("0000C9B4E3768770452E8B1B8E0232584872")
+              .checksum(new Checksum(ChecksumType.ADLER32.createMessageDigest()))
               .build();
 
         var request = mock(FlushRequest.class);