From 652ead54e586cbd86e5e1ccc8067d596181fa4df Mon Sep 17 00:00:00 2001 From: romanetar Date: Thu, 24 Aug 2023 20:15:47 +0200 Subject: [PATCH 1/2] logs upload command Signed-off-by: romanetar --- app/Console/Commands/LogsUploader.php | 96 +++++++++++++++++++++++ app/Console/Kernel.php | 4 + config/filesystems.php | 14 ++++ tests/OAuth2SummitOrdersApiTest.php | 106 ++++++++++++++------------ 4 files changed, 170 insertions(+), 50 deletions(-) create mode 100644 app/Console/Commands/LogsUploader.php diff --git a/app/Console/Commands/LogsUploader.php b/app/Console/Commands/LogsUploader.php new file mode 100644 index 000000000..715010328 --- /dev/null +++ b/app/Console/Commands/LogsUploader.php @@ -0,0 +1,96 @@ +exists($ui_file_name)) { + $uploads_info = explode(PHP_EOL, $logs_fs->get($ui_file_name)); + sort($uploads_info); + $first_date_str = $uploads_info[0]; + $date_from = Carbon::createFromFormat($date_format, $first_date_str); + //get upload gaps + $period = CarbonPeriod::create($date_from, $yesterday); + + foreach ($period as $date) { + $date_str = $date->format($date_format); + if (!in_array($date_str, $uploads_info)) { + $pending_uploads[] = $this->formatFileName($date_str); + } + } + } else { + $pending_uploads[] = $this->formatFileName($yesterday->format($date_format)); + } + + foreach ($pending_uploads as $pending_upload) { + $logs_fs->append($ui_file_name, $pending_upload); + } + + //Storage::disk('logs_s3')->put($log_name, $content); + } catch (Exception $ex) { + Log::error($ex); + } + } +} diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 9610df662..95ab8cc85 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -49,6 +49,7 @@ class Kernel extends ConsoleKernel \App\Console\Commands\PresentationMediaUploadsRegenerateTemporalLinks::class, \App\Console\Commands\PurgeAuditLogCommand::class, \App\Console\Commands\SummitBadgesQREncryptor::class, + \App\Console\Commands\LogsUploader::class, ]; /** @@ -98,5 +99,8 @@ protected function schedule(Schedule $schedule) } $schedule->command('summit:presentations-regenerate-media-uploads-temporal-public-urls')->everyMinute()->withoutOverlapping()->onOneServer(); + + // logs upload + $schedule->command('management:logs-uploader')->dailyAt("07:00")->timezone('UTC')->withoutOverlapping()->onOneServer(); } } diff --git a/config/filesystems.php b/config/filesystems.php index abe7602ea..1290d046b 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -48,6 +48,11 @@ 'root' => storage_path('app'), ], + 'logs' => [ + 'driver' => 'local', + 'root' => storage_path('logs'), + ], + 'public' => [ 'driver' => 'local', 'root' => storage_path('app/public'), @@ -92,6 +97,15 @@ 'endpoint' => env('AWS_ENDPOINT_ASSETS'), ], + 'logs_s3' => [ + 'driver' => 's3', + 'key' => env('AWS_ACCESS_KEY_ID_LOGS'), + 'secret' => env('AWS_SECRET_ACCESS_KEY_LOGS'), + 'region' => env('AWS_DEFAULT_REGION_LOGS'), + 'bucket' => env('AWS_BUCKET_LOGS'), + 'endpoint' => env('AWS_ENDPOINT_LOGS'), + ], + 'static_images_s3' => [ 'driver' => 's3', 'key' => env('AWS_ACCESS_KEY_ID_STATIC_IMAGES'), diff --git a/tests/OAuth2SummitOrdersApiTest.php b/tests/OAuth2SummitOrdersApiTest.php index c052d6aa2..fe948e891 100644 --- a/tests/OAuth2SummitOrdersApiTest.php +++ b/tests/OAuth2SummitOrdersApiTest.php @@ -13,10 +13,12 @@ **/ use Illuminate\Support\Facades\App; +use LaravelDoctrine\ORM\Facades\EntityManager; use models\summit\PaymentGatewayProfileFactory; use models\summit\IPaymentConstants; use App\Models\Foundation\Summit\Factories\SummitTicketTypeFactory; use App\Models\Foundation\Summit\Factories\SummitBadgeTypeFactory; +use models\summit\Summit; use services\model\ISummitService; /** @@ -58,54 +60,54 @@ final class OAuth2SummitOrdersApiTest extends ProtectedApiTest use InsertOrdersTestData; - protected function setUp():void - { - parent::setUp(); - self::$test_secret_key = env('TEST_STRIPE_SECRET_KEY'); - self::$test_public_key = env('TEST_STRIPE_PUBLISHABLE_KEY'); - self::$live_secret_key = env('LIVE_STRIPE_SECRET_KEY'); - self::$live_public_key = env('LIVE_STRIPE_PUBLISHABLE_KEY'); - - self::insertSummitTestData(); - self::InsertOrdersTestData(); - // build payment profile and attach to summit - self::$profile = PaymentGatewayProfileFactory::build(IPaymentConstants::ProviderStripe, [ - 'application_type' => IPaymentConstants::ApplicationTypeRegistration, - 'is_test_mode' => true, - 'test_publishable_key' => self::$test_public_key, - 'test_secret_key' => self::$test_secret_key, - 'is_active' => false, - ]); - - // build default badge type - - $defaultBadge = SummitBadgeTypeFactory::build([ - 'name' => 'DEFAULT', - 'is_default' => true, - ]); - - // build ticket type - - self::$ticketType = SummitTicketTypeFactory::build(self::$summit, [ - 'name' => 'TICKET_1', - 'cost' => 100, - 'quantity_2_sell' => 1000, - ]); - - self::$summit->addPaymentProfile(self::$profile); - self::$summit->addBadgeType($defaultBadge); - self::$summit->addTicketType(self::$ticketType); - - self::$em->persist(self::$summit); - self::$em->flush(); - - } - - protected function tearDown():void - { - self::clearSummitTestData(); - parent::tearDown(); - } +// protected function setUp():void +// { +// parent::setUp(); +// self::$test_secret_key = env('TEST_STRIPE_SECRET_KEY'); +// self::$test_public_key = env('TEST_STRIPE_PUBLISHABLE_KEY'); +// self::$live_secret_key = env('LIVE_STRIPE_SECRET_KEY'); +// self::$live_public_key = env('LIVE_STRIPE_PUBLISHABLE_KEY'); +// +// self::insertSummitTestData(); +// self::InsertOrdersTestData(); +// // build payment profile and attach to summit +// self::$profile = PaymentGatewayProfileFactory::build(IPaymentConstants::ProviderStripe, [ +// 'application_type' => IPaymentConstants::ApplicationTypeRegistration, +// 'is_test_mode' => true, +// 'test_publishable_key' => self::$test_public_key, +// 'test_secret_key' => self::$test_secret_key, +// 'is_active' => false, +// ]); +// +// // build default badge type +// +// $defaultBadge = SummitBadgeTypeFactory::build([ +// 'name' => 'DEFAULT', +// 'is_default' => true, +// ]); +// +// // build ticket type +// +// self::$ticketType = SummitTicketTypeFactory::build(self::$summit, [ +// 'name' => 'TICKET_1', +// 'cost' => 100, +// 'quantity_2_sell' => 1000, +// ]); +// +// self::$summit->addPaymentProfile(self::$profile); +// self::$summit->addBadgeType($defaultBadge); +// self::$summit->addTicketType(self::$ticketType); +// +// self::$em->persist(self::$summit); +// self::$em->flush(); +// +// } +// +// protected function tearDown():void +// { +// self::clearSummitTestData(); +// parent::tearDown(); +// } /** * @return mixed @@ -380,11 +382,14 @@ public function testReserveWithSummit(){ $res = memory_get_peak_usage(true); + $summit_repository = EntityManager::getRepository(Summit::class); + self::$summit = $summit_repository->find(3685); + $summitId = self::$summit->getId(); $companyId = 5; $service = App::make(ISummitService::class); - $service->addCompany($summitId, $companyId); + //$service->addCompany($summitId, $companyId); $company = self::$summit->getRegistrationCompanyById($companyId); $params = [ @@ -398,7 +403,8 @@ public function testReserveWithSummit(){ "owner_company" => $company->getName(), "owner_company_id" => $company->getId(), "tickets" => [ - ["type_id" => self::$ticketType->getId()], + //["type_id" => self::$ticketType->getId()], + ["type_id" => 3080], ] ]; From 27d908bcc2a39f91adb00681a3fa4f8952215f50 Mon Sep 17 00:00:00 2001 From: romanetar Date: Tue, 29 Aug 2023 16:30:48 +0200 Subject: [PATCH 2/2] logs upload command Signed-off-by: romanetar --- .env.example | 13 ++- app/Console/Commands/LogsUploader.php | 56 ++++--------- app/Services/BaseServicesProvider.php | 8 ++ .../FileSystem/ILogsUploadService.php | 25 ++++++ app/Services/FileSystem/LogsUploadService.php | 83 +++++++++++++++++++ config/log.php | 2 + config/server.php | 1 + 7 files changed, 146 insertions(+), 42 deletions(-) create mode 100644 app/Services/FileSystem/ILogsUploadService.php create mode 100644 app/Services/FileSystem/LogsUploadService.php diff --git a/.env.example b/.env.example index 5650df0ca..99f8bc90f 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,7 @@ APP_SCOPE_BASE_REALM=http://localhost APP_OAUTH_2_0_CLIENT_ID=clientid APP_OAUTH_2_0_CLIENT_SECRET=clientsecret APP_OAUTH_2_0_AUTH_SERVER_BASE_URL=http://localhost +APP_NODE_NBR= DB_HOST=localhost DB_DATABASE=homestead @@ -185,4 +186,14 @@ AWS_ENDPOINT= REGISTRATION_ORDER_PUBLIC_EDIT_TTL=10 DEFAULT_PROFILE_IMAGE= SCHEDULE_USE_REALTIME_UPDATE=1 -SAMSUNG_REGISTRATION_API_ENDPOINT= \ No newline at end of file +SAMSUNG_REGISTRATION_API_ENDPOINT= + +# Remote logs storage +APP_LOG_FILE_NAME_PREFIX=laravel +REMOTE_LOGS_STORAGE_NAME='logs_s3' + +AWS_ACCESS_KEY_ID_LOGS= +AWS_SECRET_ACCESS_KEY_LOGS= +AWS_DEFAULT_REGION_LOGS= +AWS_ENDPOINT_LOGS= +AWS_BUCKET_LOGS= \ No newline at end of file diff --git a/app/Console/Commands/LogsUploader.php b/app/Console/Commands/LogsUploader.php index 715010328..4af41b4aa 100644 --- a/app/Console/Commands/LogsUploader.php +++ b/app/Console/Commands/LogsUploader.php @@ -12,12 +12,8 @@ * limitations under the License. **/ -use Carbon\Carbon; -use Carbon\CarbonPeriod; -use Exception; +use App\Services\FileSystem\ILogsUploadService; use Illuminate\Console\Command; -use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\Storage; /** * Class LogsUploader @@ -47,50 +43,28 @@ final class LogsUploader extends Command */ protected $description = 'Upload local logs to external storage'; - private function formatFileName(string $date_str) { - return "laravel-{$date_str}.log"; + /** + * @var ILogsUploadService + */ + protected $logs_upload_service; + + /** + * SummitEventSetAvgRateProcessor constructor. + * @param ILogsUploadService $logs_upload_service + */ + public function __construct(ILogsUploadService $logs_upload_service) + { + parent::__construct(); + $this->logs_upload_service = $logs_upload_service; } /** * Execute the console command. * * @return mixed - * @throws \Illuminate\Contracts\Filesystem\FileNotFoundException */ public function handle() { - try { - $date_format = 'Y-m-d'; - $ui_file_name = 'uploads_info'; - $pending_uploads = []; - $yesterday = Carbon::yesterday('UTC'); - - $logs_fs = Storage::disk('logs'); - if ($logs_fs->exists($ui_file_name)) { - $uploads_info = explode(PHP_EOL, $logs_fs->get($ui_file_name)); - sort($uploads_info); - $first_date_str = $uploads_info[0]; - $date_from = Carbon::createFromFormat($date_format, $first_date_str); - //get upload gaps - $period = CarbonPeriod::create($date_from, $yesterday); - - foreach ($period as $date) { - $date_str = $date->format($date_format); - if (!in_array($date_str, $uploads_info)) { - $pending_uploads[] = $this->formatFileName($date_str); - } - } - } else { - $pending_uploads[] = $this->formatFileName($yesterday->format($date_format)); - } - - foreach ($pending_uploads as $pending_upload) { - $logs_fs->append($ui_file_name, $pending_upload); - } - - //Storage::disk('logs_s3')->put($log_name, $content); - } catch (Exception $ex) { - Log::error($ex); - } + $this->logs_upload_service->startUpload(); } } diff --git a/app/Services/BaseServicesProvider.php b/app/Services/BaseServicesProvider.php index bedf08760..c0c42d841 100644 --- a/app/Services/BaseServicesProvider.php +++ b/app/Services/BaseServicesProvider.php @@ -23,6 +23,8 @@ use App\Services\Apis\PasswordlessAPI; use App\Services\Apis\Samsung\ISamsungRegistrationAPI; use App\Services\Apis\Samsung\SamsungRegistrationAPI; +use App\Services\FileSystem\ILogsUploadService; +use App\Services\FileSystem\LogsUploadService; use App\Services\Model\FolderService; use App\Services\Model\IFolderService; use App\Services\utils\EmailExcerptService; @@ -148,6 +150,11 @@ function(){ ); } ); + + App::singleton( + ILogsUploadService::class, + LogsUploadService::class + ); } /** @@ -172,6 +179,7 @@ public function provides() ILockManagerService::class, IPasswordlessAPI::class, ISamsungRegistrationAPI::class, + ILogsUploadService::class, ]; } } \ No newline at end of file diff --git a/app/Services/FileSystem/ILogsUploadService.php b/app/Services/FileSystem/ILogsUploadService.php new file mode 100644 index 000000000..4e4487ce5 --- /dev/null +++ b/app/Services/FileSystem/ILogsUploadService.php @@ -0,0 +1,25 @@ +format($date_format)}.log"; + } + + /** + * Returns the remote log file path based on the local name and the node number. + */ + private function formatRemoteLogFilePath(string $local_name, string $node_number): string + { + return "{$node_number}/{$local_name}.gz"; + } + + /** + * @inheritdoc + */ + public function startUpload(): void + { + try { + $node_number = Config::get("server.app_node_number"); + $remote_storage_name = Config::get("log.remote_storage_name"); + + $today_log_file_name = $this->formatFileName(Carbon::today('UTC')); + + $logs_fs = Storage::disk('logs'); + $remote_logs_fs = Storage::disk($remote_storage_name); + + $local_log_files = array_filter($logs_fs->allFiles(), function ($file_name) use ($today_log_file_name) { + return $file_name != $today_log_file_name && str_ends_with($file_name, '.log'); + }); + sort($local_log_files); + + foreach ($local_log_files as $local_log_file) { + $content = $logs_fs->get($local_log_file); + if (empty($content)) continue; + + $remote_file_path = $this->formatRemoteLogFilePath($local_log_file, $node_number); + if ($remote_logs_fs->exists($remote_file_path)) continue; + + $deflateContext = deflate_init(ZLIB_ENCODING_GZIP); + $compressed = deflate_add($deflateContext, $content, ZLIB_FINISH); + + $remote_logs_fs->put($remote_file_path, $compressed); + + $logs_fs->put("{$local_log_file}.gz", $compressed); + $logs_fs->delete($local_log_file); + } + } catch (Exception $ex) { + Log::error($ex); + } + } +} \ No newline at end of file diff --git a/config/log.php b/config/log.php index 4090dc4a4..ea6244f74 100644 --- a/config/log.php +++ b/config/log.php @@ -11,4 +11,6 @@ 'level' => env('LOG_LEVEL', 'error'), 'email_level' => env('LOG_EMAIL_LEVEL', 'error'), 'email_subject' => env('LOG_EMAIL_SUBJECT', ''), + 'remote_storage_name' => env('REMOTE_LOGS_STORAGE_NAME', 'logs_s3'), + 'app_log_file_name_prefix' => env('APP_LOG_FILE_NAME_PREFIX', 'laravel'), ]; \ No newline at end of file diff --git a/config/server.php b/config/server.php index 87820eaf2..19f0ae7ad 100644 --- a/config/server.php +++ b/config/server.php @@ -26,4 +26,5 @@ 'ss_encrypt_cypher' => env('SS_ENCRYPT_CYPHER', ''), 'google_geocoding_api_key' => env('GOOGLE_GEO_CODING_API_KEY', ''), 'samsung_registration_api_endpoint' => env('SAMSUNG_REGISTRATION_API_ENDPOINT', ''), + 'app_node_number' => env('APP_NODE_NBR', 1), ); \ No newline at end of file