New service, initial version: FileSorter

This is a service which auto sorts files based on type and timestamps. Could potentially later be enhanced to also consider meta-data native to specific types Note that while its a service, currently the dir watching inbox functionality is not yet implemented and it only performs a bootup sorting Also fixed: - issue in FileSystem class where getting the file list recursively was using the wrong variable so sub-dirs would get messed up - deleted rubbish test code in main.cpp of ProcessMetrics
LiberatorUSA · Jul 7, 2024 · eff5b7c · eff5b7c
1 parent 7a85e6d
commit eff5b7c
Show file tree

Hide file tree

Showing 37 changed files with 2,816 additions and 45 deletions.
diff --git a/.github/workflows/GUCEF_exe_FileSorter.yml b/.github/workflows/GUCEF_exe_FileSorter.yml
@@ -0,0 +1,53 @@
+name: target_GUCEF_exe_FileSorter
+
+on:
+  workflow_dispatch:
+    inputs:
+      archiveBuildArtifacts:
+        description: 'Archive Build Artifacts?'
+        required: false
+        default: 'false'
+      targetPlatform:
+        description: 'Target platform to build?'
+        required: false
+        default: 'all'
+
+jobs:
+
+  GUCEF_exe_FileSorter-all:
+    runs-on: self-hosted
+    env:
+      TARGET_NAME: GUCEF_exe_FileSorter
+      TARGET_PLATFORM: all
+      TARGET_PROJECT: projects/CMake/targets/GUCEF_exe_FileSorter
+      PRODUCT_NAME: FileSorter
+      CI_TOOL: GITHUB-ACTIONS
+      BUILD_TOOL: CMAKE
+    steps:
+      - uses: actions/checkout@master
+      - name: Build FileSorter for platform all
+        run: ./projects/CI/targets/ci_build.sh
+      - name: Archive Debian build artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: Debian packages
+          path: |
+            ./common/bin/GUCEF_exe_FileSorter-all/packages/*FileSorter*.deb
+      - name: Archive RPM build artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: RPM packages
+          path: |
+            ./common/bin/GUCEF_exe_FileSorter-all/packages/*FileSorter*.rpm
+      - name: Archive Tarball - bz2 compressed build artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: tar.bz2 packages
+          path: |
+            ./common/bin/GUCEF_exe_FileSorter-all/packages/*FileSorter*.tar.bz2
+      - name: Archive Tarball - Xz compressed build artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: tar.xz packages
+          path: |
+            ./common/bin/GUCEF_exe_FileSorter-all/packages/*FileSorter*.tar.xz
diff --git a/README.md b/README.md
@@ -128,6 +128,7 @@ If you get into trouble for whatever reason with stale/bad output files (disk is
 - pubsub2storage: Adapter service which translates more generally between messaging paradigms and storage. Will be superseded by pubsub2pubsub.
 - pubsub2pubsub: Adapter service which translates more generally between different messaging paradigms. Will supersede many of the other adapter services.
 - FilePusher: Agent service which monitors the file system for certain files and pushes them to a VFS destination for example AWS S3.
+- FileSorter: Agent service which automatically sorts files. Handy for initial sort for digital picture and video archive for example.
 - GucefLogService: Service which can accept GUCEF platform logs streamed via network connection. Aimed at situations where we cannot have/access local logs
 - ServerPortExtender: Service used to reverse inbound/outbound connection initiation for an application server port thus bypassing egress only restrictions
 - UdpTransformer: Service which takes ingress UDP and performs a simplistic transform followed by a retransmission

diff --git a/platform/gucefCORE/include/gucefCORE_CDate.h b/platform/gucefCORE/include/gucefCORE_CDate.h
@@ -74,6 +74,8 @@ class GUCEF_CORE_PUBLIC_CPP CDate : public CIDate
 
     virtual UInt8 GetMonth( void ) const GUCEF_VIRTUAL_OVERRIDE;
 
+    virtual CORE::CString GetMonthName( void ) const GUCEF_VIRTUAL_OVERRIDE;
+
     bool SetDay( UInt8 day );
 
     virtual UInt8 GetDay( void ) const GUCEF_VIRTUAL_OVERRIDE;

diff --git a/platform/gucefCORE/include/gucefCORE_CDateTime.h b/platform/gucefCORE/include/gucefCORE_CDateTime.h
@@ -150,6 +150,10 @@ class GUCEF_CORE_PUBLIC_CPP CDateTime : public CDate ,
 
     void Clear( void );
 
+    void AddSeconds( Int32 secondsToAdd );
+    void AddMinutes( Int32 minutesToAdd );
+    void AddHours( Int32 hoursToAdd );
+
     explicit CDateTime( const struct tm* src, bool isUtc );
 
     explicit CDateTime( const time_t src, bool isUtc );

diff --git a/platform/gucefCORE/include/gucefCORE_CIDate.h b/platform/gucefCORE/include/gucefCORE_CIDate.h
@@ -31,6 +31,11 @@
 #define GUCEF_CORE_MACROS_H
 #endif /* GUCEF_CORE_MACROS_H ? */
 
+#ifndef GUCEF_CORE_CSTRING_H
+#include "gucefCORE_CString.h"
+#define GUCEF_CORE_CSTRING_H
+#endif /* GUCEF_CORE_CSTRING_H ? */
+
 /*-------------------------------------------------------------------------//
 //                                                                         //
 //      NAMESPACE                                                          //
@@ -54,6 +59,8 @@ class GUCEF_CORE_PUBLIC_CPP CIDate
 
     virtual UInt8 GetMonth( void ) const = 0;
 
+    virtual CORE::CString GetMonthName( void ) const = 0;
+
     virtual UInt8 GetDay( void ) const = 0;
 };
 

diff --git a/platform/gucefCORE/include/gucefCORE_CResourceMetaData.h b/platform/gucefCORE/include/gucefCORE_CResourceMetaData.h
@@ -81,7 +81,10 @@ class GUCEF_CORE_PUBLIC_CPP CResourceMetaData
 
     CResourceMetaData( void );
     CResourceMetaData( const CResourceMetaData& src );
+
     void Clear( void );
+
+    bool GetEarliestUtcDt( CORE::CDateTime& earliest ) const;
 };
 
 /*-------------------------------------------------------------------------//

diff --git a/platform/gucefCORE/src/gucefCORE_CDate.cpp b/platform/gucefCORE/src/gucefCORE_CDate.cpp
@@ -226,6 +226,20 @@ CDate::GetMonth( void ) const
 
 /*-------------------------------------------------------------------------*/
 
+CORE::CString
+CDate::GetMonthName( void ) const
+{GUCEF_TRACE;
+
+    static const char* monthNames[] = { "January", "February", "March", "April", "May", "June",
+                                        "July", "August", "September", "October", "November", "December" };
+    if ( m_month >= 1 && m_month <= 12 )
+        return monthNames[ m_month-1 ];
+    else
+        return "Unknown";
+}
+
+/*-------------------------------------------------------------------------*/
+
 bool 
 CDate::SetDay( UInt8 day )
 {GUCEF_TRACE;

diff --git a/platform/gucefCORE/src/gucefCORE_CDateTime.cpp b/platform/gucefCORE/src/gucefCORE_CDateTime.cpp
@@ -1290,6 +1290,42 @@ CDateTime::Clear( void )
     *this = Empty;
 }
 
+/*-------------------------------------------------------------------------*/
+
+void
+CDateTime::AddSeconds( Int32 secondsToAdd )
+{GUCEF_TRACE;
+
+    UInt64 baseTimestamp = ToUnixEpochBasedTicksInMillisecs();
+    Int64 msDelta = (Int64) secondsToAdd * 1000;
+    baseTimestamp += msDelta;
+    FromUnixEpochBasedTicksInMillisecs( baseTimestamp );    
+}
+
+/*-------------------------------------------------------------------------*/
+
+void
+CDateTime::AddMinutes( Int32 minutesToAdd )
+{GUCEF_TRACE;
+
+    UInt64 baseTimestamp = ToUnixEpochBasedTicksInMillisecs();
+    Int64 msDelta = (Int64) minutesToAdd * 60 * 1000;
+    baseTimestamp += msDelta;
+    FromUnixEpochBasedTicksInMillisecs( baseTimestamp );    
+}
+
+/*-------------------------------------------------------------------------*/
+
+void
+CDateTime::AddHours( Int32 hoursToAdd )
+{GUCEF_TRACE;
+
+    UInt64 baseTimestamp = ToUnixEpochBasedTicksInMillisecs();
+    Int64 msDelta = (Int64) hoursToAdd * 60 * 60 * 1000;
+    baseTimestamp += msDelta;
+    FromUnixEpochBasedTicksInMillisecs( baseTimestamp );    
+}
+
 /*-------------------------------------------------------------------------//
 //                                                                         //
 //      NAMESPACE                                                          //

diff --git a/platform/gucefCORE/src/gucefCORE_CResourceMetaData.cpp b/platform/gucefCORE/src/gucefCORE_CResourceMetaData.cpp
@@ -99,6 +99,77 @@ CResourceMetaData::Clear( void )
     resourceExists = false;
 }
 
+/*-------------------------------------------------------------------------*/
+
+bool
+CResourceMetaData::GetEarliestUtcDt( CORE::CDateTime& earliest ) const
+{GUCEF_TRACE;
+
+    if ( hasCreationDateTime && hasModifiedDateTime && hasLastAccessedDateTime )
+    {
+        if ( creationDateTime > modifiedDateTime )
+        {
+            if ( modifiedDateTime > lastAccessedDateTime )
+            {
+                earliest = lastAccessedDateTime.ToUTC();
+                return true;
+            }
+            else
+            {
+                earliest = modifiedDateTime.ToUTC();
+                return true;
+            }
+        }
+        else
+        {
+            if ( creationDateTime > lastAccessedDateTime )
+            {
+                earliest = lastAccessedDateTime.ToUTC();
+                return true;
+            }
+            else
+            {
+                earliest = creationDateTime.ToUTC();
+                return true;
+            }
+        }
+    }
+    else
+    if ( hasCreationDateTime && hasModifiedDateTime )
+    {
+        if ( creationDateTime > modifiedDateTime )
+        {
+            earliest = modifiedDateTime.ToUTC();
+            return true;
+        }
+        else
+        {
+            earliest = creationDateTime.ToUTC();
+            return true;
+        }
+    }
+    else 
+    if ( hasCreationDateTime )
+    {
+        earliest = creationDateTime.ToUTC();
+        return true;
+    }
+    else 
+    if ( hasModifiedDateTime )
+    {
+        earliest = modifiedDateTime.ToUTC();
+        return true;
+    }
+    else 
+    if ( hasLastAccessedDateTime )
+    {
+        earliest = lastAccessedDateTime.ToUTC();
+        return true;
+    }
+
+    return false;
+}
+
 /*-------------------------------------------------------------------------//
 //                                                                         //
 //      NAMESPACE                                                          //

diff --git a/platform/gucefVFS/src/gucefVFS_CFileSystemArchive.cpp b/platform/gucefVFS/src/gucefVFS_CFileSystemArchive.cpp
@@ -268,7 +268,6 @@ CFileSystemArchive::GetListFromRoot( const CORE::CString& actualFsDir  ,
 
     CORE::CString vfsPath = CORE::CombinePath( vfsMountLocation, vfsArchiveLocation );
 
-    CORE::CString filename;
     struct CORE::SDI_Data* did = CORE::DI_First_Dir_Entry( actualFsDir.C_String() );
     if ( did != NULL )
     {
@@ -282,7 +281,7 @@ CFileSystemArchive::GetListFromRoot( const CORE::CString& actualFsDir  ,
             {
                 if ( addFiles )
                 {
-                    filename = CORE::DI_Name( did );
+                    CORE::CString filename = CORE::DI_Name( did );
                     if ( filename != '.' && filename != ".." )
                     {
                         if ( CVFS::FilterValidation( filename ,
@@ -318,9 +317,8 @@ CFileSystemArchive::GetListFromRoot( const CORE::CString& actualFsDir  ,
 
                     if ( recursive )
                     {
-
                         CORE::CString fsSubdir = CORE::CombinePath( actualFsDir, dirName );
-                        CORE::CString vfsSubdir = CORE::CombinePath( vfsArchiveLocation, filename );
+                        CORE::CString vfsSubdir = CORE::CombinePath( vfsArchiveLocation, dirName );
 
                         // Recursively process the sub-dir
                         GetListFromRoot( fsSubdir              ,

diff --git a/premake4.lua b/premake4.lua
@@ -185,6 +185,7 @@ solution( "GUCEF" )
   include( "tools/FileDiff" )
   include( "tools/FilePusher" )
   include( "tools/FileReplacer" )
+  include( "tools/FileSorter" )
   include( "tools/GucefArchiver" )
   include( "tools/GucefLogServiceApp" )
   include( "tools/GucefLogServiceClientPlugin" )

diff --git a/projects/Android/targets/GUCEF/Android.mk b/projects/Android/targets/GUCEF/Android.mk
@@ -411,6 +411,9 @@ include $(MY_MODULE_PATH)/Android.mk
 MY_MODULE_PATH := $(PROJECT_ROOT_PATH)/../../../../tools/FilePusher
 include $(MY_MODULE_PATH)/Android.mk
 
+MY_MODULE_PATH := $(PROJECT_ROOT_PATH)/../../../../tools/FileSorter
+include $(MY_MODULE_PATH)/Android.mk
+
 MY_MODULE_PATH := $(PROJECT_ROOT_PATH)/../../../../tools/ProcessMetrics
 include $(MY_MODULE_PATH)/Android.mk