Has Cache use last modified time as metadata. (#1031)

Author: coollog

jib-core/src/main/java/com/google/cloud/tools/jib/ncache/Cache.java

@@ -23,14 +23,15 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 
 /**
  * Cache for storing data to be shared between Jib executions.
  *
- * <p>Uses the default cache storage engine ({@link DefaultCacheStorage}) and layer entries as the
- * selector ({@link LayerEntriesSelector}).
+ * <p>Uses the default cache storage engine ({@link DefaultCacheStorage}), layer entries as the
+ * selector ({@link LayerEntriesSelector}), and last modified time as the metadata.
  *
  * <p>This class is immutable and safe to use across threads.
  */
@@ -56,8 +57,8 @@ private Cache(CacheStorage cacheStorage) {
   }
 
   /**
-   * Saves a cache entry with only a layer {@link Blob}. Use {@link #write(Blob, ImmutableList,
-   * Blob)} to include a selector and metadata.
+   * Saves a cache entry with only a layer {@link Blob}. Use {@link #write(Blob, ImmutableList)} to
+   * include a selector and metadata.
    *
    * @param layerBlob the layer {@link Blob}
    * @return the {@link CacheEntry} for the written layer
@@ -73,19 +74,22 @@ public CacheEntry write(Blob layerBlob) throws IOException {
    *
    * @param layerBlob the layer {@link Blob}
    * @param layerEntries the layer entries that make up the layer
-   * @param metadataBlob the metadata {@link Blob}
    * @return the {@link CacheEntry} for the written layer and metadata
    * @throws IOException if an I/O exception occurs
    */
-  public CacheEntry write(Blob layerBlob, ImmutableList<LayerEntry> layerEntries, Blob metadataBlob)
+  public CacheEntry write(Blob layerBlob, ImmutableList<LayerEntry> layerEntries)
       throws IOException {
     return cacheStorage.write(
         CacheWrite.withSelectorAndMetadata(
-            layerBlob, LayerEntriesSelector.generateSelector(layerEntries), metadataBlob));
+            layerBlob,
+            LayerEntriesSelector.generateSelector(layerEntries),
+            LastModifiedTimeMetadata.generateMetadata(layerEntries)));
   }
 
   /**
-   * Retrieves the {@link CacheEntry} that was built from the {@code layerEntries}.
+   * Retrieves the {@link CacheEntry} that was built from the {@code layerEntries}. The last
+   * modified time of the {@code layerEntries} must match the last modified time as stored by the
+   * metadata of the {@link CacheEntry}.
    *
    * @param layerEntries the layer entries to match against
    * @return a {@link CacheEntry} that was built from {@code layerEntries}, if found
@@ -100,7 +104,28 @@ public Optional<CacheEntry> retrieve(ImmutableList<LayerEntry> layerEntries)
       return Optional.empty();
     }
 
-    return cacheStorage.retrieve(optionalSelectedLayerDigest.get());
+    Optional<CacheEntry> optionalCacheEntry =
+        cacheStorage.retrieve(optionalSelectedLayerDigest.get());
+    if (!optionalCacheEntry.isPresent()) {
+      return Optional.empty();
+    }
+
+    CacheEntry cacheEntry = optionalCacheEntry.get();
+
+    Optional<FileTime> optionalRetrievedLastModifiedTime =
+        LastModifiedTimeMetadata.getLastModifiedTime(cacheEntry);
+    if (!optionalRetrievedLastModifiedTime.isPresent()) {
+      return Optional.empty();
+    }
+
+    FileTime retrievedLastModifiedTime = optionalRetrievedLastModifiedTime.get();
+    FileTime expectedLastModifiedTime = LastModifiedTimeMetadata.getLastModifiedTime(layerEntries);
+
+    if (!expectedLastModifiedTime.equals(retrievedLastModifiedTime)) {
+      return Optional.empty();
+    }
+
+    return Optional.of(cacheEntry);
   }
 
   /**

jib-core/src/main/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadata.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2018 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.ncache;
+
+import com.google.cloud.tools.jib.blob.Blob;
+import com.google.cloud.tools.jib.blob.Blobs;
+import com.google.cloud.tools.jib.image.LayerEntry;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
+import java.util.Optional;
+
+/**
+ * Serializes/deserializes metadata storing the latest last modified time of all the source files in
+ * {@link LayerEntry}s for a layer.
+ *
+ * <p>Use {@link #generateMetadata} to serialize the latest last modified time of all the source
+ * files in {@link LayerEntry}s for a layer into a {@link Blob} containing the serialized last
+ * modified time. Use {@link #getLastModifiedTime(CacheEntry)} to deserialize the metadata in a
+ * {@link CacheEntry} into a last modified time.
+ */
+class LastModifiedTimeMetadata {
+
+  /**
+   * Generates the metadata {@link Blob} for the list of {@link LayerEntry}s. The metadata is the
+   * latest last modified time of all the source files in the list of {@link LayerEntry}s serialized
+   * using {@link Instant#toString}.
+   *
+   * @param layerEntries the list of {@link LayerEntry}s
+   * @return the generated metadata
+   */
+  static Blob generateMetadata(ImmutableList<LayerEntry> layerEntries) throws IOException {
+    return Blobs.from(getLastModifiedTime(layerEntries).toInstant().toString());
+  }
+
+  /**
+   * Gets the latest last modified time of all the source files in the list of {@link LayerEntry}s.
+   *
+   * @param layerEntries the list of {@link LayerEntry}s
+   * @return the last modified time
+   */
+  static FileTime getLastModifiedTime(ImmutableList<LayerEntry> layerEntries) throws IOException {
+    FileTime maxLastModifiedTime = FileTime.from(Instant.MIN);
+
+    for (LayerEntry layerEntry : layerEntries) {
+      FileTime lastModifiedTime = Files.getLastModifiedTime(layerEntry.getSourceFile());
+      if (lastModifiedTime.compareTo(maxLastModifiedTime) > 0) {
+        maxLastModifiedTime = lastModifiedTime;
+      }
+    }
+
+    return maxLastModifiedTime;
+  }
+
+  /**
+   * Gets the last modified time from the metadata of a {@link CacheEntry}.
+   *
+   * @param cacheEntry the {@link CacheEntry}
+   * @return the last modified time, if the metadata is present
+   * @throws IOException if deserialization of the metadata failed
+   */
+  static Optional<FileTime> getLastModifiedTime(CacheEntry cacheEntry) throws IOException {
+    if (!cacheEntry.getMetadataBlob().isPresent()) {
+      return Optional.empty();
+    }
+
+    Blob metadataBlob = cacheEntry.getMetadataBlob().get();
+    return Optional.of(FileTime.from(Instant.parse(Blobs.writeToString(metadataBlob))));
+  }
+
+  private LastModifiedTimeMetadata() {}
+}

jib-core/src/test/java/com/google/cloud/tools/jib/ncache/CacheTest.java

@@ -27,8 +27,10 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 import org.junit.Assert;
@@ -104,35 +106,38 @@ private static long sizeOf(Blob blob) throws IOException {
   private DescriptorDigest layerDiffId1;
   private long layerSize1;
   private ImmutableList<LayerEntry> layerEntries1;
-  private Blob metadataBlob1;
 
   private Blob layerBlob2;
   private DescriptorDigest layerDigest2;
   private DescriptorDigest layerDiffId2;
   private long layerSize2;
   private ImmutableList<LayerEntry> layerEntries2;
-  private Blob metadataBlob2;
 
   @Before
   public void setUp() throws IOException {
+    Path directory = temporaryFolder.newFolder().toPath();
+    Files.createDirectory(directory.resolve("source"));
+    Files.createFile(directory.resolve("source/file"));
+    Files.createDirectories(directory.resolve("another/source"));
+    Files.createFile(directory.resolve("another/source/file"));
+
     layerBlob1 = Blobs.from("layerBlob1");
     layerDigest1 = digestOf(compress(layerBlob1));
     layerDiffId1 = digestOf(layerBlob1);
     layerSize1 = sizeOf(compress(layerBlob1));
     layerEntries1 =
         ImmutableList.of(
-            new LayerEntry(Paths.get("source/file"), AbsoluteUnixPath.get("/extraction/path")),
             new LayerEntry(
-                Paths.get("another/source/file"),
+                directory.resolve("source/file"), AbsoluteUnixPath.get("/extraction/path")),
+            new LayerEntry(
+                directory.resolve("another/source/file"),
                 AbsoluteUnixPath.get("/another/extraction/path")));
-    metadataBlob1 = Blobs.from("metadata");
 
     layerBlob2 = Blobs.from("layerBlob2");
     layerDigest2 = digestOf(compress(layerBlob2));
     layerDiffId2 = digestOf(layerBlob2);
     layerSize2 = sizeOf(compress(layerBlob2));
     layerEntries2 = ImmutableList.of();
-    metadataBlob2 = Blobs.from("metadata");
   }
 
   @Test
@@ -159,31 +164,36 @@ public void testWriteLayerOnly_retrieveByLayerDigest()
   }
 
   @Test
-  public void testWriteWithSelectorAndMetadata_retrieveByLayerDigest()
+  public void testWriteWithLayerEntries_retrieveByLayerDigest()
       throws IOException, CacheCorruptedException {
     Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath());
 
-    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1));
+    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1));
     verifyIsLayer1WithMetadata(cache.retrieve(layerDigest1).orElseThrow(AssertionError::new));
     Assert.assertFalse(cache.retrieve(layerDigest2).isPresent());
   }
 
   @Test
-  public void testWriteWithSelectorAndMetadata_retrieveByLayerEntries()
+  public void testWriteWithLayerEntries_retrieveByLayerEntries()
       throws IOException, CacheCorruptedException {
     Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath());
 
-    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1));
+    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1));
     verifyIsLayer1WithMetadata(cache.retrieve(layerEntries1).orElseThrow(AssertionError::new));
     Assert.assertFalse(cache.retrieve(layerDigest2).isPresent());
+
+    // A source file modification results in the cached layer to be out-of-date and not retrieved.
+    Files.setLastModifiedTime(
+        layerEntries1.get(0).getSourceFile(), FileTime.from(Instant.now().plusSeconds(1)));
+    Assert.assertFalse(cache.retrieve(layerEntries1).isPresent());
   }
 
   @Test
   public void testRetrieveWithTwoEntriesInCache() throws IOException, CacheCorruptedException {
     Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath());
 
-    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1));
-    verifyIsLayer2WithMetadata(cache.write(layerBlob2, layerEntries2, metadataBlob2));
+    verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1));
+    verifyIsLayer2WithMetadata(cache.write(layerBlob2, layerEntries2));
     verifyIsLayer1WithMetadata(cache.retrieve(layerDigest1).orElseThrow(AssertionError::new));
     verifyIsLayer2WithMetadata(cache.retrieve(layerDigest2).orElseThrow(AssertionError::new));
     verifyIsLayer1WithMetadata(cache.retrieve(layerEntries1).orElseThrow(AssertionError::new));
@@ -212,7 +222,9 @@ private void verifyIsLayer1NoMetadata(CacheEntry cacheEntry) throws IOException
   private void verifyIsLayer1WithMetadata(CacheEntry cacheEntry) throws IOException {
     verifyIsLayer1(cacheEntry);
     Assert.assertTrue(cacheEntry.getMetadataBlob().isPresent());
-    Assert.assertEquals("metadata", Blobs.writeToString(cacheEntry.getMetadataBlob().get()));
+    Assert.assertEquals(
+        Blobs.writeToString(LastModifiedTimeMetadata.generateMetadata(layerEntries1)),
+        Blobs.writeToString(cacheEntry.getMetadataBlob().get()));
   }
 
   /**
@@ -242,6 +254,8 @@ private void verifyIsLayer2WithMetadata(CacheEntry cacheEntry) throws IOExceptio
     Assert.assertEquals(layerDiffId2, cacheEntry.getLayerDiffId());
     Assert.assertEquals(layerSize2, cacheEntry.getLayerSize());
     Assert.assertTrue(cacheEntry.getMetadataBlob().isPresent());
-    Assert.assertEquals("metadata", Blobs.writeToString(cacheEntry.getMetadataBlob().get()));
+    Assert.assertEquals(
+        Blobs.writeToString(LastModifiedTimeMetadata.generateMetadata(layerEntries2)),
+        Blobs.writeToString(cacheEntry.getMetadataBlob().get()));
   }
 }