Fix #3 Implement face features extraction

modules/core/src/main/scala/fr/janalyse/sotohp/core/PhotoStream.scala

@@ -11,7 +11,10 @@ object PhotoStream {
   type PhotoLazyStream   = ZStream[PhotoStoreService, PhotoStreamIssues, LazyPhoto]
 
   def photoStream(photoOwnerId: PhotoOwnerId): PhotoStream = {
-    ???
+    PhotoStoreService
+      .photoStateStream()
+      .filter(_.photoOwnerId == photoOwnerId) // TODO refactor required - performance issue
+      .mapZIO(state => PhotoOperations.makePhotoFromStoredState(state))
   }
 
   def photoStream(): PhotoStream = {
@@ -21,7 +24,9 @@ object PhotoStream {
   }
 
   def photoLazyStream(photoOwnerId: PhotoOwnerId): PhotoLazyStream = {
-    ???
+    PhotoStoreService
+      .photoLazyStream()
+      .filter(_.state.photoOwnerId == photoOwnerId) // TODO refactor required - performance issue
   }
 
   def photoLazyStream(): PhotoLazyStream = {

modules/core/src/main/scala/fr/janalyse/sotohp/store/PhotoStoreService.scala

@@ -70,25 +70,31 @@ trait PhotoStoreService {
   // photo classifications collection
   def photoClassificationsGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoClassifications]]
   def photoClassificationsContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean]
-  def photoClassificationsUpsert(photoId: PhotoId, normalized: PhotoClassifications): IO[PhotoStoreIssue, Unit]
+  def photoClassificationsUpsert(photoId: PhotoId, photoClassifications: PhotoClassifications): IO[PhotoStoreIssue, Unit]
   def photoClassificationsDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit]
 
   // photo objects collection
   def photoObjectsGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoObjects]]
   def photoObjectsContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean]
-  def photoObjectsUpsert(photoId: PhotoId, normalized: PhotoObjects): IO[PhotoStoreIssue, Unit]
+  def photoObjectsUpsert(photoId: PhotoId, photoObjects: PhotoObjects): IO[PhotoStoreIssue, Unit]
   def photoObjectsDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit]
 
   // photo faces collection
   def photoFacesGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoFaces]]
   def photoFacesContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean]
-  def photoFacesUpsert(photoId: PhotoId, normalized: PhotoFaces): IO[PhotoStoreIssue, Unit]
+  def photoFacesUpsert(photoId: PhotoId, photoFaces: PhotoFaces): IO[PhotoStoreIssue, Unit]
   def photoFacesDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit]
 
+  // photo face features collection
+  def photoFaceFeaturesGet(faceId: FaceId): IO[PhotoStoreIssue, Option[FaceFeatures]]
+  def photoFaceFeaturesContains(faceId: FaceId): IO[PhotoStoreIssue, Boolean]
+  def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): IO[PhotoStoreIssue, Unit]
+  def photoFaceFeaturesDelete(faceId: FaceId): IO[PhotoStoreIssue, Unit]
+
   // photo description collection
   def photoDescriptionGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoDescription]]
   def photoDescriptionContains(photoId: PhotoId): IO[PhotoStoreIssue, Boolean]
-  def photoDescriptionUpsert(photoId: PhotoId, normalized: PhotoDescription): IO[PhotoStoreIssue, Unit]
+  def photoDescriptionUpsert(photoId: PhotoId, photoDescription: PhotoDescription): IO[PhotoStoreIssue, Unit]
   def photoDescriptionDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit]
 
 }
@@ -97,7 +103,7 @@ object PhotoStoreService {
   def photoLazyStream(): ZStream[PhotoStoreService, PhotoStoreIssue, LazyPhoto] = ZStream.serviceWithStream(_.photoLazyStream())
 
   def photoDelete(photoId: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoDelete(photoId))
-  
+
   def photoFirst(): ZIO[PhotoStoreService, PhotoStoreIssue, Option[LazyPhoto]]                   = serviceWithZIO(_.photoFirst())
   def photoNext(after: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Option[LazyPhoto]]      = serviceWithZIO(_.photoNext(after))
   def photoPrevious(before: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Option[LazyPhoto]] = serviceWithZIO(_.photoPrevious(before))
@@ -150,6 +156,11 @@ object PhotoStoreService {
   def photoFacesUpsert(photoId: PhotoId, metaData: PhotoFaces): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoFacesUpsert(photoId, metaData))
   def photoFacesDelete(photoId: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Unit]                       = serviceWithZIO(_.photoFacesDelete(photoId))
 
+  def photoFaceFeaturesGet(faceId: FaceId): ZIO[PhotoStoreService, PhotoStoreIssue, Option[FaceFeatures]]                = serviceWithZIO(_.photoFaceFeaturesGet(faceId))
+  def photoFaceFeaturesContains(faceId: FaceId): ZIO[PhotoStoreService, PhotoStoreIssue, Boolean]                        = serviceWithZIO(_.photoFaceFeaturesContains(faceId))
+  def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoFaceFeaturesUpsert(faceId, faceFeatures))
+  def photoFaceFeaturesDelete(faceId: FaceId): ZIO[PhotoStoreService, PhotoStoreIssue, Unit]                             = serviceWithZIO(_.photoFaceFeaturesDelete(faceId))
+
   def photoDescriptionGet(photoId: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Option[PhotoDescription]]            = serviceWithZIO(_.photoDescriptionGet(photoId))
   def photoDescriptionContains(photoId: PhotoId): ZIO[PhotoStoreService, PhotoStoreIssue, Boolean]                        = serviceWithZIO(_.photoDescriptionContains(photoId))
   def photoDescriptionUpsert(photoId: PhotoId, metaData: PhotoDescription): ZIO[PhotoStoreService, PhotoStoreIssue, Unit] = serviceWithZIO(_.photoDescriptionUpsert(photoId, metaData))

modules/core/src/main/scala/fr/janalyse/sotohp/store/PhotoStoreServiceLive.scala

@@ -22,6 +22,7 @@ trait PhotoStoreCollections {
   val photoClassificationsCollectionName = "photo-classifications"
   val photoObjectsCollectionName         = "photo-objects"
   val photoFacesCollectionName           = "photo-faces"
+  val photoFaceFeaturesCollectionName    = "photo-face-features"
   val photoDescriptionsCollectionName    = "photo-descriptions"
 
   val allCollections = List(
@@ -34,6 +35,7 @@ trait PhotoStoreCollections {
     photoClassificationsCollectionName,
     photoObjectsCollectionName,
     photoFacesCollectionName,
+    photoFaceFeaturesCollectionName,
     photoDescriptionsCollectionName
   )
 }
@@ -49,6 +51,7 @@ class PhotoStoreServiceLive private (
   classificationsCollection: LMDBCollection[DaoPhotoClassifications],
   objectsCollection: LMDBCollection[DaoPhotoObjects],
   facesCollection: LMDBCollection[DaoPhotoFaces],
+  faceFeaturesCollection: LMDBCollection[DaoFaceFeatures],
   descriptionsCollection: LMDBCollection[DaoPhotoDescription]
 ) extends PhotoStoreService {
 
@@ -58,6 +61,7 @@ class PhotoStoreServiceLive private (
   }
 
   private def photoIdToCollectionKey(photoId: PhotoId): String          = photoId.id.toString
+  private def faceIdToCollectionKey(faceId: FaceId): String             = faceId.id.toString
   private def originalIdToCollectionKey(originalId: OriginalId): String = originalId.id.toString
 
   // ===================================================================================================================
@@ -67,6 +71,7 @@ class PhotoStoreServiceLive private (
       state <- photoStateGet(photoId).some.orElseFail[PhotoStoreIssue](PhotoStoreNotFoundIssue(s"$photoId not found"))
       _     <- photoClassificationsDelete(photoId)
       _     <- photoDescriptionDelete(photoId)
+      // TODO add photo face features deletion
       _     <- photoFacesDelete(photoId)
       _     <- photoMetaDataDelete(photoId)
       _     <- photoMiniaturesDelete(photoId)
@@ -473,7 +478,7 @@ class PhotoStoreServiceLive private (
               width = that.box.width,
               height = that.box.height
             ),
-            faceId = ULID.fromString(that.faceId)
+            faceId = FaceId(ULID.fromString(that.faceId))
           )
         )
       )
@@ -518,6 +523,58 @@ class PhotoStoreServiceLive private (
       .delete(photoIdToCollectionKey(photoId))
       .mapBoth(convertFailures, _ => ())
 
+  // ===================================================================================================================
+
+  def daoFaceFeaturesToFaces(from: Option[DaoFaceFeatures]): Option[FaceFeatures] = {
+    from.map(daoFaceFeatures =>
+      FaceFeatures(
+        photoId = PhotoId(ULID.fromString(daoFaceFeatures.photoId)),
+        someoneId = daoFaceFeatures.someoneId.map(id => SomeoneId(ULID.fromString(id))),
+        box = BoundingBox(
+          x = daoFaceFeatures.box.x,
+          y = daoFaceFeatures.box.y,
+          width = daoFaceFeatures.box.width,
+          height = daoFaceFeatures.box.height
+        ),
+        features = daoFaceFeatures.features
+      )
+    )
+  }
+
+  def faceFeaturesToDaoFaces(from: FaceFeatures): DaoFaceFeatures = {
+    DaoFaceFeatures(
+      photoId = from.photoId.id.toString(),
+      someoneId = from.someoneId.map(_.toString),
+      box = DaoBoundingBox(
+        x = from.box.x,
+        y = from.box.y,
+        width = from.box.width,
+        height = from.box.height
+      ),
+      features = from.features
+    )
+  }
+
+  def photoFaceFeaturesGet(faceId: FaceId): IO[PhotoStoreIssue, Option[FaceFeatures]] =
+    faceFeaturesCollection
+      .fetch(faceIdToCollectionKey(faceId))
+      .mapBoth(convertFailures, daoFaceFeaturesToFaces)
+
+  def photoFaceFeaturesContains(faceId: FaceId): IO[PhotoStoreIssue, Boolean] =
+    faceFeaturesCollection
+      .contains(faceIdToCollectionKey(faceId))
+      .mapError(convertFailures)
+
+  def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): IO[PhotoStoreIssue, Unit] =
+    faceFeaturesCollection
+      .upsertOverwrite(faceIdToCollectionKey(faceId), faceFeaturesToDaoFaces(faceFeatures))
+      .mapBoth(convertFailures, _ => ())
+
+  def photoFaceFeaturesDelete(faceId: FaceId): IO[PhotoStoreIssue, Unit] =
+    faceFeaturesCollection
+      .delete(faceIdToCollectionKey(faceId))
+      .mapBoth(convertFailures, _ => ())
+
   // ===================================================================================================================
   def daoDescriptionToDescription(from: DaoPhotoDescription): PhotoDescription = {
     PhotoDescription(
@@ -571,6 +628,7 @@ object PhotoStoreServiceLive extends PhotoStoreCollections {
     classificationsCollection <- lmdb.collectionGet[DaoPhotoClassifications](photoClassificationsCollectionName)
     objectsCollection         <- lmdb.collectionGet[DaoPhotoObjects](photoObjectsCollectionName)
     facesCollection           <- lmdb.collectionGet[DaoPhotoFaces](photoFacesCollectionName)
+    faceFeaturesCollection    <- lmdb.collectionGet[DaoFaceFeatures](photoFaceFeaturesCollectionName)
     descriptionsCollection    <- lmdb.collectionGet[DaoPhotoDescription](photoDescriptionsCollectionName)
   } yield PhotoStoreServiceLive(
     lmdb,
@@ -583,6 +641,7 @@ object PhotoStoreServiceLive extends PhotoStoreCollections {
     classificationsCollection,
     objectsCollection,
     facesCollection,
+    faceFeaturesCollection,
     descriptionsCollection
   )
 }

modules/core/src/main/scala/fr/janalyse/sotohp/store/dao/DaoFaceFeatures.scala

@@ -0,0 +1,10 @@
+package fr.janalyse.sotohp.store.dao
+
+import zio.json.JsonCodec
+
+case class DaoFaceFeatures(
+  photoId: String,
+  someoneId: Option[String],
+  box: DaoBoundingBox,
+  features: Array[Float]
+) derives JsonCodec

modules/core/src/test/scala/fr/janalyse/sotohp/core/PhotoStoreServiceFake.scala

@@ -16,6 +16,7 @@ class PhotoStoreServiceFake(
   classificationsCollectionRef: Ref[Map[PhotoId, PhotoClassifications]],
   objectsCollectionRef: Ref[Map[PhotoId, PhotoObjects]],
   facesCollectionRef: Ref[Map[PhotoId, PhotoFaces]],
+  faceFeaturesCollectionRef: Ref[Map[FaceId, FaceFeatures]],
   descriptionsCollectionRef: Ref[Map[PhotoId, PhotoDescription]]
 ) extends PhotoStoreService {
 
@@ -24,16 +25,17 @@ class PhotoStoreServiceFake(
   override def photoDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit] = {
     for {
       state <- photoStateGet(photoId).some.orElseFail[PhotoStoreIssue](PhotoStoreNotFoundIssue(s"$photoId not found"))
-      _ <- photoClassificationsDelete(photoId)
-      _ <- photoDescriptionDelete(photoId)
-      _ <- photoFacesDelete(photoId)
-      _ <- photoMetaDataDelete(photoId)
-      _ <- photoMiniaturesDelete(photoId)
-      _ <- photoNormalizedDelete(photoId)
-      _ <- photoObjectsDelete(photoId)
-      _ <- photoPlaceDelete(photoId)
-      _ <- photoSourceDelete(state.originalId)
-      _ <- photoStateDelete(photoId)
+      _     <- photoClassificationsDelete(photoId)
+      _     <- photoDescriptionDelete(photoId)
+      // TODO add photo face features deletion
+      _     <- photoFacesDelete(photoId)
+      _     <- photoMetaDataDelete(photoId)
+      _     <- photoMiniaturesDelete(photoId)
+      _     <- photoNormalizedDelete(photoId)
+      _     <- photoObjectsDelete(photoId)
+      _     <- photoPlaceDelete(photoId)
+      _     <- photoSourceDelete(state.originalId)
+      _     <- photoStateDelete(photoId)
     } yield ()
   }
 
@@ -198,6 +200,21 @@ class PhotoStoreServiceFake(
   override def photoFacesDelete(photoId: PhotoId): IO[PhotoStoreIssue, Unit] =
     facesCollectionRef.update(collection => collection.removed(photoId))
 
+  // ===================================================================================================================
+  override def photoFaceFeaturesGet(faceId: FaceId): IO[PhotoStoreIssue, Option[FaceFeatures]] = for {
+    collection <- faceFeaturesCollectionRef.get
+  } yield collection.get(faceId)
+
+  override def photoFaceFeaturesContains(faceId: FaceId): IO[PhotoStoreIssue, Boolean] = for {
+    collection <- faceFeaturesCollectionRef.get
+  } yield collection.contains(faceId)
+
+  override def photoFaceFeaturesUpsert(faceId: FaceId, faceFeatures: FaceFeatures): IO[PhotoStoreIssue, Unit] =
+    faceFeaturesCollectionRef.update(_.updated(faceId, faceFeatures))
+
+  override def photoFaceFeaturesDelete(faceId: FaceId): IO[PhotoStoreIssue, Unit] =
+    faceFeaturesCollectionRef.update(collection => collection.removed(faceId))
+
   // ===================================================================================================================
   override def photoDescriptionGet(photoId: PhotoId): IO[PhotoStoreIssue, Option[PhotoDescription]] = for {
     collection <- descriptionsCollectionRef.get
@@ -237,6 +254,7 @@ object PhotoStoreServiceFake extends PhotoStoreCollections {
       classificationsCollection <- Ref.make(Map.empty[PhotoId, PhotoClassifications])
       objectsCollection         <- Ref.make(Map.empty[PhotoId, PhotoObjects])
       facesCollection           <- Ref.make(Map.empty[PhotoId, PhotoFaces])
+      faceFeaturesCollection    <- Ref.make(Map.empty[FaceId, FaceFeatures])
       descriptionsCollection    <- Ref.make(Map.empty[PhotoId, PhotoDescription])
     } yield PhotoStoreServiceFake(
       statesCollection,
@@ -248,6 +266,7 @@ object PhotoStoreServiceFake extends PhotoStoreCollections {
       classificationsCollection,
       objectsCollection,
       facesCollection,
+      faceFeaturesCollection,
       descriptionsCollection
     )
   )

modules/model/src/main/scala/fr/janalyse/sotohp/model/FaceFeatures.scala

@@ -0,0 +1,10 @@
+package fr.janalyse.sotohp.model
+
+import wvlet.airframe.ulid.ULID
+
+case class FaceFeatures(
+  photoId: PhotoId,
+  someoneId: Option[SomeoneId],
+  box: BoundingBox,
+  features: Array[Float]
+)

modules/model/src/main/scala/fr/janalyse/sotohp/model/PhotoFaces.scala

@@ -2,8 +2,14 @@ package fr.janalyse.sotohp.model
 
 import wvlet.airframe.ulid.ULID
 
+case class FaceId(
+  id: ULID
+) extends AnyVal {
+  override def toString(): String = id.toString
+}
+
 case class DetectedFace(
-  faceId: ULID,
+  faceId: FaceId,
   someoneId: Option[SomeoneId],
   box: BoundingBox
 )

modules/processor/src/main/java/fr/janalyse/sotohp/processor/FeatureComparison.java

@@ -0,0 +1,60 @@
+package fr.janalyse.sotohp.processor;
+
+/*
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+// code coming from : https://github.com/deepjavalibrary/djl/blob/master/examples/src/main/java/ai/djl/examples/inference/face/FeatureComparison.java
+
+import ai.djl.ModelException;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.translate.TranslateException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public final class FeatureComparison {
+
+    private static final Logger logger = LoggerFactory.getLogger(FeatureComparison.class);
+
+    private FeatureComparison() {}
+
+    public static void main(String[] args) throws IOException, ModelException, TranslateException {
+        Path imageFile1 = Paths.get("src/test/resources/kana1.jpg");
+        Image img1 = ImageFactory.getInstance().fromFile(imageFile1);
+        Path imageFile2 = Paths.get("src/test/resources/kana2.jpg");
+        Image img2 = ImageFactory.getInstance().fromFile(imageFile2);
+
+        float[] feature1 = FeatureExtraction.predict(img1);
+        float[] feature2 = FeatureExtraction.predict(img2);
+
+        logger.info(Float.toString(calculSimilar(feature1, feature2)));
+    }
+
+    public static float calculSimilar(float[] feature1, float[] feature2) {
+        float ret = 0.0f;
+        float mod1 = 0.0f;
+        float mod2 = 0.0f;
+        int length = feature1.length;
+        for (int i = 0; i < length; ++i) {
+            ret += feature1[i] * feature2[i];
+            mod1 += feature1[i] * feature1[i];
+            mod2 += feature2[i] * feature2[i];
+        }
+        return (float) ((ret / Math.sqrt(mod1) / Math.sqrt(mod2) + 1) / 2.0f);
+    }
+}