Feature/external logs persistence

.env.example

@@ -7,6 +7,7 @@ APP_SCOPE_BASE_REALM=http://localhost
 APP_OAUTH_2_0_CLIENT_ID=clientid
 APP_OAUTH_2_0_CLIENT_SECRET=clientsecret
 APP_OAUTH_2_0_AUTH_SERVER_BASE_URL=http://localhost
+APP_NODE_NBR=
 
 DB_HOST=localhost
 DB_DATABASE=homestead
@@ -185,4 +186,14 @@ AWS_ENDPOINT=
 REGISTRATION_ORDER_PUBLIC_EDIT_TTL=10
 DEFAULT_PROFILE_IMAGE=
 SCHEDULE_USE_REALTIME_UPDATE=1
-SAMSUNG_REGISTRATION_API_ENDPOINT=
+SAMSUNG_REGISTRATION_API_ENDPOINT=
+
+# Remote logs storage
+APP_LOG_FILE_NAME_PREFIX=laravel
+REMOTE_LOGS_STORAGE_NAME='logs_s3'
+
+AWS_ACCESS_KEY_ID_LOGS=
+AWS_SECRET_ACCESS_KEY_LOGS=
+AWS_DEFAULT_REGION_LOGS=
+AWS_ENDPOINT_LOGS=
+AWS_BUCKET_LOGS=

app/Console/Commands/LogsUploader.php

@@ -0,0 +1,70 @@
+<?php namespace App\Console\Commands;
+/**
+ * Copyright 2023 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+use App\Services\FileSystem\ILogsUploadService;
+use Illuminate\Console\Command;
+
+/**
+ * Class LogsUploader
+ * @package App\Console\Commands
+ */
+final class LogsUploader extends Command
+{
+    /**
+     * The console command name.
+     *
+     * @var string
+     */
+    protected $name = 'management:logs-uploader';
+
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'management:logs-uploader';
+
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Upload local logs to external storage';
+
+    /**
+     * @var ILogsUploadService
+     */
+    protected $logs_upload_service;
+
+    /**
+     * SummitEventSetAvgRateProcessor constructor.
+     * @param ILogsUploadService $logs_upload_service
+     */
+    public function __construct(ILogsUploadService $logs_upload_service)
+    {
+        parent::__construct();
+        $this->logs_upload_service = $logs_upload_service;
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $this->logs_upload_service->startUpload();
+    }
+}

app/Console/Kernel.php

@@ -49,6 +49,7 @@ class Kernel extends ConsoleKernel
         \App\Console\Commands\PresentationMediaUploadsRegenerateTemporalLinks::class,
         \App\Console\Commands\PurgeAuditLogCommand::class,
         \App\Console\Commands\SummitBadgesQREncryptor::class,
+        \App\Console\Commands\LogsUploader::class,
     ];
 
     /**
@@ -98,5 +99,8 @@ protected function schedule(Schedule $schedule)
         }
 
         $schedule->command('summit:presentations-regenerate-media-uploads-temporal-public-urls')->everyMinute()->withoutOverlapping()->onOneServer();
+
+        // logs upload
+        $schedule->command('management:logs-uploader')->dailyAt("07:00")->timezone('UTC')->withoutOverlapping()->onOneServer();
     }
 }

app/Services/BaseServicesProvider.php

@@ -23,6 +23,8 @@
 use App\Services\Apis\PasswordlessAPI;
 use App\Services\Apis\Samsung\ISamsungRegistrationAPI;
 use App\Services\Apis\Samsung\SamsungRegistrationAPI;
+use App\Services\FileSystem\ILogsUploadService;
+use App\Services\FileSystem\LogsUploadService;
 use App\Services\Model\FolderService;
 use App\Services\Model\IFolderService;
 use App\Services\utils\EmailExcerptService;
@@ -148,6 +150,11 @@ function(){
                 );
             }
         );
+
+        App::singleton(
+            ILogsUploadService::class,
+            LogsUploadService::class
+        );
     }
 
     /**
@@ -172,6 +179,7 @@ public function provides()
             ILockManagerService::class,
             IPasswordlessAPI::class,
             ISamsungRegistrationAPI::class,
+            ILogsUploadService::class,
         ];
     }
 }

app/Services/FileSystem/ILogsUploadService.php

@@ -0,0 +1,25 @@
+<?php namespace App\Services\FileSystem;
+/**
+ * Copyright 2023 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+/**
+ * Interface ILogsUploadService
+ * @package App\Services\FileSystem
+ */
+interface ILogsUploadService
+{
+    /**
+     * @return void
+     */
+    public function startUpload(): void;
+}

app/Services/FileSystem/LogsUploadService.php

@@ -0,0 +1,83 @@
+<?php namespace App\Services\FileSystem;
+
+/**
+ * Copyright 2023 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+use Carbon\Carbon;
+use Exception;
+use Illuminate\Support\Facades\Config;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+/**
+ * Class LogsUploadService
+ * @package App\Services\Model
+ */
+final class LogsUploadService
+    implements ILogsUploadService
+{
+    /**
+     * Returns the log file name based on the date parameter.
+     */
+    private function formatFileName(Carbon $date): string
+    {
+        $app_log_file_name_prefix = Config::get("log.app_log_file_name_prefix");
+        $date_format = 'Y-m-d';
+        return "{$app_log_file_name_prefix}-{$date->format($date_format)}.log";
+    }
+
+    /**
+     * Returns the remote log file path based on the local name and the node number.
+     */
+    private function formatRemoteLogFilePath(string $local_name, string $node_number): string
+    {
+        return "{$node_number}/{$local_name}.gz";
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function startUpload(): void
+    {
+        try {
+            $node_number = Config::get("server.app_node_number");
+            $remote_storage_name = Config::get("log.remote_storage_name");
+
+            $today_log_file_name = $this->formatFileName(Carbon::today('UTC'));
+
+            $logs_fs = Storage::disk('logs');
+            $remote_logs_fs = Storage::disk($remote_storage_name);
+
+            $local_log_files = array_filter($logs_fs->allFiles(), function ($file_name) use ($today_log_file_name) {
+                return $file_name != $today_log_file_name && str_ends_with($file_name, '.log');
+            });
+            sort($local_log_files);
+
+            foreach ($local_log_files as $local_log_file) {
+                $content = $logs_fs->get($local_log_file);
+                if (empty($content)) continue;
+
+                $remote_file_path = $this->formatRemoteLogFilePath($local_log_file, $node_number);
+                if ($remote_logs_fs->exists($remote_file_path)) continue;
+
+                $deflateContext = deflate_init(ZLIB_ENCODING_GZIP);
+                $compressed = deflate_add($deflateContext, $content, ZLIB_FINISH);
+
+                $remote_logs_fs->put($remote_file_path, $compressed);
+
+                $logs_fs->put("{$local_log_file}.gz", $compressed);
+                $logs_fs->delete($local_log_file);
+            }
+        } catch (Exception $ex) {
+            Log::error($ex);
+        }
+    }
+}

config/filesystems.php

@@ -48,6 +48,11 @@
             'root' => storage_path('app'),
         ],
 
+        'logs' => [
+            'driver' => 'local',
+            'root' => storage_path('logs'),
+        ],
+
         'public' => [
             'driver' => 'local',
             'root' => storage_path('app/public'),
@@ -92,6 +97,15 @@
             'endpoint' => env('AWS_ENDPOINT_ASSETS'),
         ],
 
+        'logs_s3' => [
+            'driver' => 's3',
+            'key' => env('AWS_ACCESS_KEY_ID_LOGS'),
+            'secret' => env('AWS_SECRET_ACCESS_KEY_LOGS'),
+            'region' => env('AWS_DEFAULT_REGION_LOGS'),
+            'bucket' => env('AWS_BUCKET_LOGS'),
+            'endpoint' => env('AWS_ENDPOINT_LOGS'),
+        ],
+
         'static_images_s3' => [
             'driver' => 's3',
             'key' => env('AWS_ACCESS_KEY_ID_STATIC_IMAGES'),

config/log.php

@@ -11,4 +11,6 @@
     'level'         => env('LOG_LEVEL', 'error'),
     'email_level'   => env('LOG_EMAIL_LEVEL', 'error'),
     'email_subject' => env('LOG_EMAIL_SUBJECT', ''),
+    'remote_storage_name' => env('REMOTE_LOGS_STORAGE_NAME', 'logs_s3'),
+    'app_log_file_name_prefix' => env('APP_LOG_FILE_NAME_PREFIX', 'laravel'),
 ];

config/server.php

@@ -26,4 +26,5 @@
     'ss_encrypt_cypher'                 => env('SS_ENCRYPT_CYPHER', ''),
     'google_geocoding_api_key'          => env('GOOGLE_GEO_CODING_API_KEY', ''),
     'samsung_registration_api_endpoint' => env('SAMSUNG_REGISTRATION_API_ENDPOINT', ''),
+    'app_node_number'                   => env('APP_NODE_NBR', 1),
 );