117682: Filter-media last date

atmire · Sep 11, 2024 · f1efc2c · f1efc2c
1 parent e4ebbf0
commit f1efc2c
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 12 deletions.
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterScript.java
@@ -8,8 +8,10 @@
 package org.dspace.app.mediafilter;
 
 import java.time.LocalDate;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -223,7 +225,7 @@ public void internalRun() throws Exception {
         }
 
         if (fromDate != null) {
-            mediaFilterService.setFromDate(fromDate);
+            mediaFilterService.setFromDate(Date.from(fromDate.atStartOfDay(ZoneId.systemDefault()).toInstant()));
         }
 
         Context c = null;

diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/MediaFilterServiceImpl.java
@@ -9,8 +9,6 @@
 
 import java.io.InputStream;
 import java.sql.SQLException;
-import java.time.LocalDate;
-import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -37,6 +35,7 @@
 import org.dspace.content.service.CollectionService;
 import org.dspace.content.service.CommunityService;
 import org.dspace.content.service.ItemService;
+import org.dspace.content.service.SiteService;
 import org.dspace.core.Constants;
 import org.dspace.core.Context;
 import org.dspace.core.SelfNamedPlugin;
@@ -75,6 +74,8 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
     @Autowired(required = true)
     protected ItemService itemService;
     @Autowired(required = true)
+    protected SiteService siteService;
+    @Autowired(required = true)
     protected ConfigurationService configurationService;
 
     protected DSpaceRunnableHandler handler;
@@ -96,7 +97,7 @@ public class MediaFilterServiceImpl implements MediaFilterService, InitializingB
     protected boolean isVerbose = false;
     protected boolean isQuiet = false;
     protected boolean isForce = false; // default to not forced
-    protected LocalDate fromDate = null;
+    protected Date fromDate = null;
 
     protected MediaFilterServiceImpl() {
 
@@ -116,6 +117,14 @@ public void afterPropertiesSet() throws Exception {
 
     @Override
     public void applyFiltersAllItems(Context context) throws Exception {
+        boolean storeLastDate = false;
+        if (fromDate == null) {
+            storeLastDate = true;
+            String lastDate = siteService.getMetadata(siteService.findSite(context), "dc.filtermedia.lastdate");
+            if (lastDate != null) {
+                fromDate = new DCDate(lastDate).toDate();
+            }
+        }
         if (skipList != null) {
             //if a skip-list exists, we need to filter community-by-community
             //so we can respect what is in the skip-list
@@ -128,7 +137,7 @@ public void applyFiltersAllItems(Context context) throws Exception {
             Iterator<Item> itemIterator =
                     itemService.findByLastModifiedSince(
                             context,
-                            Date.from(fromDate.atStartOfDay(ZoneId.systemDefault()).toInstant())
+                            fromDate
                     );
             while (itemIterator.hasNext() && processed < max2Process) {
                 applyFiltersItem(context, itemIterator.next());
@@ -140,6 +149,14 @@ public void applyFiltersAllItems(Context context) throws Exception {
                 applyFiltersItem(context, itemIterator.next());
             }
         }
+        if (storeLastDate) {
+            siteService.clearMetadata(context, siteService.findSite(context),
+                "dc", "filtermedia", "lastdate", Item.ANY);
+            siteService.addMetadata(context, siteService.findSite(context),
+                "dc", "filtermedia", "lastdate", null,
+                new DCDate(new Date()).toString());
+            siteService.update(context, siteService.findSite(context));
+        }
     }
 
     @Override
@@ -172,7 +189,12 @@ public void applyFiltersCollection(Context context, Collection collection)
         collection = context.reloadEntity(collection);
         //only apply filters if collection not in skip-list
         if (!inSkipList(collection.getHandle())) {
-            Iterator<Item> itemIterator = itemService.findAllByCollection(context, collection);
+            Iterator<Item> itemIterator;
+            if (fromDate != null) {
+                itemIterator = itemService.findByCollectionLastModifiedSince(context, collection, fromDate);
+            } else {
+                itemIterator = itemService.findAllByCollection(context, collection);
+            }
             while (itemIterator.hasNext() && processed < max2Process) {
                 applyFiltersItem(context, itemIterator.next());
             }
@@ -603,7 +625,7 @@ public void setLogHandler(DSpaceRunnableHandler handler) {
     }
 
     @Override
-    public void setFromDate(LocalDate fromDate) {
+    public void setFromDate(Date fromDate) {
         this.fromDate = fromDate;
     }
 }
diff --git a/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java b/dspace-api/src/main/java/org/dspace/app/mediafilter/service/MediaFilterService.java
@@ -8,7 +8,7 @@
 package org.dspace.app.mediafilter.service;
 
 import java.sql.SQLException;
-import java.time.LocalDate;
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -151,5 +151,5 @@ public void updatePoliciesOfDerivativeBitstreams(Context context, Item item, Bit
      */
     public void setLogHandler(DSpaceRunnableHandler handler);
 
-    public void setFromDate(LocalDate fromDate);
+    public void setFromDate(Date fromDate);
 }
diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java
@@ -1691,6 +1691,12 @@ public Iterator<Item> findByLastModifiedSince(Context context, Date last)
         return itemDAO.findByLastModifiedSince(context, last);
     }
 
+    @Override
+    public Iterator<Item> findByCollectionLastModifiedSince(Context context, Collection collection, Date last)
+        throws SQLException {
+        return itemDAO.findAllByCollectionLastModifiedSince(context, collection, last);
+    }
+
     @Override
     public int countTotal(Context context) throws SQLException {
         return itemDAO.countRows(context);

diff --git a/dspace-api/src/main/java/org/dspace/content/dao/ItemDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/ItemDAO.java
@@ -132,6 +132,9 @@ Iterator<Item> findArchivedByCollectionExcludingOwning(Context context, Collecti
 
     Iterator<Item> findAllByCollection(Context context, Collection collection) throws SQLException;
 
+    Iterator<Item> findAllByCollectionLastModifiedSince(Context context, Collection collection, Date last)
+        throws SQLException;
+
     Iterator<Item> findAllByCollection(Context context, Collection collection, Integer limit, Integer offset)
         throws SQLException;
 

diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemDAOImpl.java
@@ -440,9 +440,34 @@ public int countItems(Context context, List<Collection> collections, boolean inc
     @Override
     public Iterator<Item> findByLastModifiedSince(Context context, Date since)
         throws SQLException {
-        Query query = createQuery(context,
-                "SELECT i.id FROM Item i WHERE lastModified > :last_modified ORDER BY id");
-        query.setParameter("last_modified", since, TemporalType.TIMESTAMP);
+        CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context);
+        CriteriaQuery<UUID> criteriaQuery = criteriaBuilder.createQuery(UUID.class);
+        Root<Item> itemRoot = criteriaQuery.from(Item.class);
+        criteriaQuery.select(itemRoot.get(Item_.id));
+        criteriaQuery.where(criteriaBuilder.greaterThan(itemRoot.get(Item_.lastModified), since));
+        criteriaQuery.orderBy(criteriaBuilder.asc(itemRoot.get((Item_.id))));
+
+        // Transform into a query object to execute
+        Query query = createQuery(context, criteriaQuery);
+        @SuppressWarnings("unchecked")
+        List<UUID> uuids = query.getResultList();
+        return new UUIDIterator<Item>(context, uuids, Item.class, this);
+    }
+
+    @Override
+    public Iterator<Item> findAllByCollectionLastModifiedSince(Context context, Collection collection,
+                                                               Date last) throws SQLException {
+        // Select UUID of all items which have this "collection" in their list of collections
+        CriteriaBuilder criteriaBuilder = getCriteriaBuilder(context);
+        CriteriaQuery<UUID> criteriaQuery = criteriaBuilder.createQuery(UUID.class);
+        Root<Item> itemRoot = criteriaQuery.from(Item.class);
+        criteriaQuery.select(itemRoot.get(Item_.id));
+        criteriaQuery.where(criteriaBuilder.isMember(collection, itemRoot.get(Item_.collections)));
+        criteriaQuery.where(criteriaBuilder.greaterThan(itemRoot.get(Item_.lastModified), last));
+        criteriaQuery.orderBy(criteriaBuilder.asc(itemRoot.get((Item_.id))));
+
+        // Transform into a query object to execute
+        Query query = createQuery(context, criteriaQuery);
         @SuppressWarnings("unchecked")
         List<UUID> uuids = query.getResultList();
         return new UUIDIterator<Item>(context, uuids, Item.class, this);

diff --git a/dspace-api/src/main/java/org/dspace/content/service/ItemService.java b/dspace-api/src/main/java/org/dspace/content/service/ItemService.java
@@ -856,6 +856,18 @@ Iterator<Item> findByMetadataFieldAuthority(Context context, String mdString, St
     Iterator<Item> findByLastModifiedSince(Context context, Date last)
         throws SQLException;
 
+    /**
+     * Get all the archived items in this collection, last modified since a given Date. The order is indeterminate.
+     *
+     * @param context    DSpace context object
+     * @param collection Collection (parent)
+     * @param last       Earliest interesting last-modified date.
+     * @return an iterator over the items in the collection.
+     * @throws SQLException if database error
+     */
+    Iterator<Item> findByCollectionLastModifiedSince(Context context, Collection collection, Date last)
+        throws SQLException;
+
     /**
      * counts items in the given community
      *

diff --git a/dspace/config/registries/dublin-core-types.xml b/dspace/config/registries/dublin-core-types.xml
@@ -196,6 +196,13 @@
     <scope_note>Recommend for theses/dissertations.</scope_note>
   </dc-type>
 
+  <dc-type>
+    <schema>dc</schema>
+    <element>filtermedia</element>
+    <qualifier>lastdate</qualifier>
+    <scope_note>Last date filter-media was run</scope_note>
+  </dc-type>
+
   <dc-type>
 	<schema>dc</schema>
     <element>identifier</element>