logs upload command

Signed-off-by: romanetar <[email protected]>

Author: romanetar

.env.example

@@ -185,4 +185,15 @@ AWS_ENDPOINT=
 REGISTRATION_ORDER_PUBLIC_EDIT_TTL=10
 DEFAULT_PROFILE_IMAGE=
 SCHEDULE_USE_REALTIME_UPDATE=1
-SAMSUNG_REGISTRATION_API_ENDPOINT=
+SAMSUNG_REGISTRATION_API_ENDPOINT=
+
+NODE_NBR=1
+
+# Remote logs storage
+REMOTE_LOGS_STORAGE_NAME='logs_s3'
+
+AWS_ACCESS_KEY_ID_LOGS=
+AWS_SECRET_ACCESS_KEY_LOGS=
+AWS_DEFAULT_REGION_LOGS=
+AWS_ENDPOINT_LOGS=
+AWS_BUCKET_LOGS=

app/Console/Commands/LogsUploader.php

@@ -13,9 +13,9 @@
  **/
 
 use Carbon\Carbon;
-use Carbon\CarbonPeriod;
 use Exception;
 use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Config;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
 
@@ -47,8 +47,21 @@ final class LogsUploader extends Command
      */
     protected $description = 'Upload local logs to external storage';
 
-    private function formatFileName(string $date_str) {
-        return "laravel-{$date_str}.log";
+    /**
+     * Returns the log file name based on the date parameter.
+     */
+    private function formatFileName(Carbon $date): string
+    {
+        $date_format = 'Y-m-d';
+        return "laravel-{$date->format($date_format)}.log";
+    }
+
+    /**
+     * Returns the remote log file path based on the local name and the node number.
+     */
+    private function formatRemoteLogFilePath(string $local_name, string $node_number): string
+    {
+        return "{$node_number}/{$local_name}.gz";
     }
 
     /**
@@ -60,35 +73,34 @@ private function formatFileName(string $date_str) {
     public function handle()
     {
         try {
-            $date_format = 'Y-m-d';
-            $ui_file_name = 'uploads_info';
-            $pending_uploads = [];
-            $yesterday = Carbon::yesterday('UTC');
+            $node_number = Config::get("server.node_number");
+            $remote_storage_name = Config::get("log.remote_storage_name");
+
+            $today_log_file_name = $this->formatFileName(Carbon::today('UTC'));
 
             $logs_fs = Storage::disk('logs');
-            if ($logs_fs->exists($ui_file_name)) {
-                $uploads_info = explode(PHP_EOL, $logs_fs->get($ui_file_name));
-                sort($uploads_info);
-                $first_date_str = $uploads_info[0];
-                $date_from = Carbon::createFromFormat($date_format, $first_date_str);
-                //get upload gaps
-                $period = CarbonPeriod::create($date_from, $yesterday);
-
-                foreach ($period as $date) {
-                    $date_str = $date->format($date_format);
-                    if (!in_array($date_str, $uploads_info)) {
-                        $pending_uploads[] = $this->formatFileName($date_str);
-                    }
-                }
-            } else {
-                $pending_uploads[] = $this->formatFileName($yesterday->format($date_format));
-            }
+            $remote_logs_fs = Storage::disk($remote_storage_name);
 
-            foreach ($pending_uploads as $pending_upload) {
-                $logs_fs->append($ui_file_name, $pending_upload);
-            }
+            $local_log_files = array_filter($logs_fs->allFiles(), function ($file_name) use ($today_log_file_name) {
+                return $file_name != $today_log_file_name && str_ends_with($file_name, '.log');
+            });
+            sort($local_log_files);
+
+            foreach ($local_log_files as $local_log_file) {
+                $content = $logs_fs->get($local_log_file);
+                if (empty($content)) continue;
 
-            //Storage::disk('logs_s3')->put($log_name, $content);
+                $remote_file_path = $this->formatRemoteLogFilePath($local_log_file, $node_number);
+                if ($remote_logs_fs->exists($remote_file_path)) continue;
+
+                $deflateContext = deflate_init(ZLIB_ENCODING_GZIP);
+                $compressed = deflate_add($deflateContext, $content, ZLIB_FINISH);
+
+                $remote_logs_fs->put($remote_file_path, $compressed);
+
+                $logs_fs->put("{$local_log_file}.gz", $compressed);
+                $logs_fs->delete($local_log_file);
+            }
         } catch (Exception $ex) {
             Log::error($ex);
         }

config/log.php

@@ -11,4 +11,5 @@
     'level'         => env('LOG_LEVEL', 'error'),
     'email_level'   => env('LOG_EMAIL_LEVEL', 'error'),
     'email_subject' => env('LOG_EMAIL_SUBJECT', ''),
+    'remote_storage_name' => env('REMOTE_LOGS_STORAGE_NAME', 's3'),
 ];

config/server.php

@@ -26,4 +26,5 @@
     'ss_encrypt_cypher'                 => env('SS_ENCRYPT_CYPHER', ''),
     'google_geocoding_api_key'          => env('GOOGLE_GEO_CODING_API_KEY', ''),
     'samsung_registration_api_endpoint' => env('SAMSUNG_REGISTRATION_API_ENDPOINT', ''),
+    'node_number'                       => env('NODE_NBR', 1),
 );