diff --git a/Makefile b/Makefile index bc31e989..a4de7c91 100644 --- a/Makefile +++ b/Makefile @@ -7,86 +7,89 @@ PHP_VERSION?=8.1 MYSQL_VERSION?=5.7 up: - docker-compose up -d - docker-compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done' + docker compose up -d + docker compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done' + +create-migration: + docker compose exec -T -u www-data cli-app sh -c "php craft migrate/create --plugin=craft-lilt-plugin add_job_attempts" down: - docker-compose down -v --remove-orphans + docker compose down -v --remove-orphans restart: down - docker-compose up -d + docker compose up -d cli: - docker-compose exec -u www-data cli-app sh + docker compose exec -u www-data cli-app sh root: - docker-compose exec -u root cli-app sh + docker compose exec -u root cli-app sh composer-install: - docker-compose exec -T -u root cli-app sh -c "apk add git" - docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin" - docker-compose exec -T -u root cli-app sh -c "rm -f composer.lock" - docker-compose exec -T -u root cli-app sh -c "rm -rf vendor" - docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env" - docker-compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php" - docker-compose exec -T -u www-data cli-app sh -c "php composer.phar install" + docker compose exec -T -u root cli-app sh -c "apk add git" + docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin" + docker compose exec -T -u root cli-app sh -c "rm -f composer.lock" + docker compose exec -T -u root cli-app sh -c "rm -rf vendor" + docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env" + docker compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php" + docker compose exec -T -u www-data cli-app sh -c "php composer.phar install" quality: - docker-compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar" - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs" - docker-compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations" + docker compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs" + docker compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations" quality-fix: - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf" codecept-build: - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build" coverage-xdebug: - docker-compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html" + docker compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html" install-pcov: - docker-compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c" - docker-compose exec -T -u root cli-app sh -c "pecl install pcov || true" - docker-compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov" + docker compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c" + docker compose exec -T -u root cli-app sh -c "pecl install pcov || true" + docker compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov" coverage: install-pcov - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html" tests-with-coverage: codecept-build install-pcov unit-coverage integration-coverage functional-coverage integration-coverage: - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml" functional-coverage: - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml" unit-coverage: - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml" integration: codecept-build - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration" functional: codecept-build - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional" unit: codecept-build - docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit" + docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit" test: functional integration unit prepare-container: - PHP_VERSION=7.2 docker-compose up -d - docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin" - docker-compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git" - docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env" - docker-compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php" - docker-compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer" + PHP_VERSION=7.2 docker compose up -d + docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin" + docker compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git" + docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env" + docker compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php" + docker compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer" test-craft-versions: prepare-container - docker-compose exec -T -u www-data cli-app bash -c \ + docker compose exec -T -u www-data cli-app bash -c \ "./craft-versions.sh ${CRAFT_VERSION}" require-guzzle-v6: - docker-compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true" - docker-compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi' + docker compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true" + docker compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi' diff --git a/docker-compose.yml b/docker-compose.yml index 3820eba5..01c35244 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - services: cli-app: image: craftcms/cli:${PHP_VERSION} diff --git a/src/Craftliltplugin.php b/src/Craftliltplugin.php index abe9a4ec..8dff1d04 100644 --- a/src/Craftliltplugin.php +++ b/src/Craftliltplugin.php @@ -36,13 +36,14 @@ use lilthq\craftliltplugin\services\handlers\LoadI18NHandler; use lilthq\craftliltplugin\services\handlers\PublishDraftHandler; use lilthq\craftliltplugin\services\handlers\RefreshJobStatusHandler; +use lilthq\craftliltplugin\services\handlers\ResendJobHandler; use lilthq\craftliltplugin\services\handlers\SendJobToLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\SendTranslationToLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\StartQueueManagerHandler; use lilthq\craftliltplugin\services\handlers\SyncJobFromLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\TranslationFailedHandler; use lilthq\craftliltplugin\services\handlers\UpdateJobStatusHandler; -use lilthq\craftliltplugin\services\handlers\UpdateTranslationsConnectorIds; +use lilthq\craftliltplugin\services\handlers\ResolveTranslationsConnectorIds; use lilthq\craftliltplugin\services\listeners\ListenerRegister; use lilthq\craftliltplugin\services\mappers\LanguageMapper; use lilthq\craftliltplugin\services\providers\ConnectorConfigurationProvider; @@ -109,8 +110,9 @@ * @property CreateDraftHandler $createDraftHandler * @property CopySourceTextHandler $copySourceTextHandler * @property UpdateJobStatusHandler $updateJobStatusHandler + * @property ResendJobHandler $resendJobHandler * @property SettingsRepository $settingsRepository - * @property UpdateTranslationsConnectorIds $updateTranslationsConnectorIds + * @property ResolveTranslationsConnectorIds $resolveTranslationsConnectorIds * @property PackagistRepository $packagistRepository * @property StartQueueManagerHandler $startQueueManagerHandler * @property ServiceInitializer $serviceInitializer diff --git a/src/controllers/job/PostJobRetryController.php b/src/controllers/job/PostJobRetryController.php index 4e0a145f..7d0c41b6 100644 --- a/src/controllers/job/PostJobRetryController.php +++ b/src/controllers/job/PostJobRetryController.php @@ -46,7 +46,7 @@ public function actionInvoke(): Response sprintf('Job retried (previous Lilt Job ID: %d)', $job->liltJobId) ); - Craftliltplugin::getInstance()->sendJobToLiltConnectorHandler->__invoke($job); + Craftliltplugin::getInstance()->resendJobHandler->__invoke($job->id); } diff --git a/src/elements/Job.php b/src/elements/Job.php index 540a694b..c6fb702c 100644 --- a/src/elements/Job.php +++ b/src/elements/Job.php @@ -43,6 +43,7 @@ class Job extends Element public const STATUS_FAILED = 'failed'; public const STATUS_NEEDS_ATTENTION = 'needs-attention'; + public const MAX_JOB_ATTEMPTS = 5; public $uid; public $authorId; @@ -59,6 +60,9 @@ class Job extends Element public $dateCreated; public $dateUpdated; + public $attempt = 0; + + // @codingStandardsIgnoreStart private $_author; private $_elements; diff --git a/src/migrations/Install.php b/src/migrations/Install.php index 4dcebac4..990e2ee0 100644 --- a/src/migrations/Install.php +++ b/src/migrations/Install.php @@ -32,6 +32,7 @@ public function safeUp(): void $this->createTable(CraftliltpluginParameters::JOB_TABLE_NAME, [ 'id' => $this->primaryKey()->unsigned(), + 'attempt' => $this->integer()->notNull()->defaultValue(0), 'title' => $this->string()->null(), 'authorId' => $this->integer()->null(), 'liltJobId' => $this->integer()->null(), diff --git a/src/migrations/m240526_212007_add_job_attempts.php b/src/migrations/m240526_212007_add_job_attempts.php new file mode 100644 index 00000000..2c731f12 --- /dev/null +++ b/src/migrations/m240526_212007_add_job_attempts.php @@ -0,0 +1,39 @@ +addColumn( + CraftliltpluginParameters::JOB_TABLE_NAME, + 'attempt', + $this->integer()->unsigned()->notNull()->defaultValue(0) + ); + } + + /** + * @inheritdoc + */ + public function safeDown() + { + // Remove the 'attempt' column from the 'lilt_jobs' table + $this->dropColumn( + CraftliltpluginParameters::JOB_TABLE_NAME, + 'attempt' + ); + } +} diff --git a/src/modules/FetchJobStatusFromConnector.php b/src/modules/FetchJobStatusFromConnector.php index 1e038b32..0380b719 100644 --- a/src/modules/FetchJobStatusFromConnector.php +++ b/src/modules/FetchJobStatusFromConnector.php @@ -79,14 +79,39 @@ public function execute($queue): void || $liltJob->getStatus() === JobResponse::STATUS_FAILED; if ($isJobFailed) { - $jobRecord->status = Job::STATUS_FAILED; - Craftliltplugin::getInstance()->jobLogsRepository->create( $jobRecord->id, Craft::$app->getUser()->getId(), sprintf('Job failed, received status: %s', $liltJob->getStatus()) ); + if ($jobRecord->attempt < Job::MAX_JOB_ATTEMPTS) { + // Retry job + $jobRecord->attempt++; + $jobRecord->save(); + + Craftliltplugin::getInstance()->resendJobHandler->__invoke( + $jobRecord->id + ); + + + Craftliltplugin::getInstance()->jobLogsRepository->create( + $jobRecord->id, + Craft::$app->getUser()->getId(), + sprintf( + 'Trying again to send job, attempt: %d/%d', + $jobRecord->attempt, + Job::MAX_JOB_ATTEMPTS + ) + ); + + $mutex->release($mutexKey); + $this->markAsDone($queue); + return; + } + + // Job failed, set status to failed + $jobRecord->status = Job::STATUS_FAILED; $jobRecord->save(); TranslationRecord::updateAll( @@ -108,7 +133,7 @@ public function execute($queue): void } if (!$isJobFinished) { - $queueDisableAutomaticSync = (bool) Craftliltplugin::getInstance()->settingsRepository->get( + $queueDisableAutomaticSync = (bool)Craftliltplugin::getInstance()->settingsRepository->get( SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC ); @@ -132,15 +157,12 @@ public function execute($queue): void return; } - $connectorTranslations = Craftliltplugin::getInstance()->connectorTranslationRepository->findByJobId( - $job->liltJobId - ); - + $connectorTranslations = Craftliltplugin::getInstance()->resolveTranslationsConnectorIds->update($job); $connectorTranslationsStatuses = array_map( function (TranslationResponse $connectorTranslation) { return $connectorTranslation->getStatus(); }, - $connectorTranslations->getResults() + $connectorTranslations ); $translationFinished = @@ -184,7 +206,7 @@ function (TranslationResponse $connectorTranslation) { return; } - $queueDisableAutomaticSync = (bool) Craftliltplugin::getInstance()->settingsRepository->get( + $queueDisableAutomaticSync = (bool)Craftliltplugin::getInstance()->settingsRepository->get( SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC ); @@ -208,7 +230,7 @@ function (TranslationResponse $connectorTranslation) { return; } - if ($jobRecord->isVerifiedFlow()) { + if ($jobRecord->isVerifiedFlow() || $jobRecord->isInstantFlow()) { #LILT_TRANSLATION_WORKFLOW_VERIFIED $jobRecord->status = Job::STATUS_IN_PROGRESS; @@ -217,58 +239,6 @@ function (TranslationResponse $connectorTranslation) { $this->jobId ); - Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job); - - foreach ($translations as $translation) { - Queue::push( - new FetchTranslationFromConnector( - [ - 'jobId' => $this->jobId, - 'liltJobId' => $this->liltJobId, - 'translationId' => $translation->id, - ] - ), - FetchTranslationFromConnector::PRIORITY, - 10, //10 seconds for first job - FetchTranslationFromConnector::TTR - ); - } - } - - if ($jobRecord->isInstantFlow()) { - #LILT_TRANSLATION_WORKFLOW_INSTANT - - if ( - $liltJob->getStatus() === JobResponse::STATUS_FAILED - || $liltJob->getStatus() === JobResponse::STATUS_CANCELED - ) { - $jobRecord->status = Job::STATUS_FAILED; - - TranslationRecord::updateAll( - ['status' => TranslationRecord::STATUS_FAILED], - ['jobId' => $jobRecord->id] - ); - - Craft::error([ - "message" => sprintf( - 'Set job %d and translations to status failed due to failed/cancel status from lilt', - $jobRecord->id - ), - "jobRecord" => $jobRecord, - ]); - - $mutex->release($mutexKey); - $this->markAsDone($queue); - - return; - } - - $translations = Craftliltplugin::getInstance()->translationRepository->findByJobId( - $this->jobId - ); - - Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job); - foreach ($translations as $translation) { Queue::push( new FetchTranslationFromConnector( diff --git a/src/modules/FetchTranslationFromConnector.php b/src/modules/FetchTranslationFromConnector.php index 5bffdf70..b003acf4 100644 --- a/src/modules/FetchTranslationFromConnector.php +++ b/src/modules/FetchTranslationFromConnector.php @@ -76,7 +76,7 @@ public function execute($queue): void } if (empty($translationRecord->connectorTranslationId)) { - Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job); + Craftliltplugin::getInstance()->resolveTranslationsConnectorIds->update($job); } $translationRecord->refresh(); diff --git a/src/records/JobRecord.php b/src/records/JobRecord.php index f2ac53ed..7dcfbc20 100644 --- a/src/records/JobRecord.php +++ b/src/records/JobRecord.php @@ -26,6 +26,7 @@ * @property int $sourceSiteLanguage [int(11) unsigned] * @property string $targetSiteIds [json] * @property string $dueDate [datetime] + * @property int $attempt [int(11) unsigned] * * @property-read ActiveQueryInterface $element * @property string $translationWorkflow [varchar(50)] diff --git a/src/services/ServiceInitializer.php b/src/services/ServiceInitializer.php index 8206d499..9f9dc7a6 100644 --- a/src/services/ServiceInitializer.php +++ b/src/services/ServiceInitializer.php @@ -36,13 +36,14 @@ use lilthq\craftliltplugin\services\handlers\LoadI18NHandler; use lilthq\craftliltplugin\services\handlers\PublishDraftHandler; use lilthq\craftliltplugin\services\handlers\RefreshJobStatusHandler; +use lilthq\craftliltplugin\services\handlers\ResendJobHandler; use lilthq\craftliltplugin\services\handlers\SendJobToLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\SendTranslationToLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\StartQueueManagerHandler; use lilthq\craftliltplugin\services\handlers\SyncJobFromLiltConnectorHandler; use lilthq\craftliltplugin\services\handlers\TranslationFailedHandler; use lilthq\craftliltplugin\services\handlers\UpdateJobStatusHandler; -use lilthq\craftliltplugin\services\handlers\UpdateTranslationsConnectorIds; +use lilthq\craftliltplugin\services\handlers\ResolveTranslationsConnectorIds; use lilthq\craftliltplugin\services\listeners\ListenerRegister; use lilthq\craftliltplugin\services\mappers\LanguageMapper; use lilthq\craftliltplugin\services\providers\ConnectorConfigurationProvider; @@ -79,6 +80,7 @@ public function run(): void $pluginInstance->setComponents([ 'createJobHandler' => CreateJobHandler::class, + 'resendJobHandler' => ResendJobHandler::class, 'sendJobToLiltConnectorHandler' => SendJobToLiltConnectorHandler::class, 'copySourceTextHandler' => CopySourceTextHandler::class, 'syncJobFromLiltConnectorHandler' => SyncJobFromLiltConnectorHandler::class, @@ -93,7 +95,7 @@ public function run(): void 'createTranslationsHandler' => CreateTranslationsHandler::class, 'refreshJobStatusHandler' => RefreshJobStatusHandler::class, 'updateJobStatusHandler' => UpdateJobStatusHandler::class, - 'updateTranslationsConnectorIds' => UpdateTranslationsConnectorIds::class, + 'resolveTranslationsConnectorIds' => ResolveTranslationsConnectorIds::class, 'packagistRepository' => PackagistRepository::class, 'startQueueManagerHandler' => StartQueueManagerHandler::class, 'listenerRegister' => [ diff --git a/src/services/handlers/CreateJobHandler.php b/src/services/handlers/CreateJobHandler.php index 8cbac2e3..20d2f877 100644 --- a/src/services/handlers/CreateJobHandler.php +++ b/src/services/handlers/CreateJobHandler.php @@ -24,6 +24,7 @@ public function __invoke(CreateJobCommand $command, bool $asDraft = false): Job $job->authorId = $command->getAuthorId(); $job->title = $command->getTitle(); $job->liltJobId = null; + $job->attempt = 0; $job->status = $asDraft ? Job::STATUS_DRAFT : Job::STATUS_NEW; $job->sourceSiteId = $command->getSourceSiteId(); diff --git a/src/services/handlers/EditJobHandler.php b/src/services/handlers/EditJobHandler.php index 8a43cf1f..5a5a82b2 100644 --- a/src/services/handlers/EditJobHandler.php +++ b/src/services/handlers/EditJobHandler.php @@ -58,6 +58,7 @@ public function __invoke(EditJobCommand $command): Job $job->elementIds = $command->getEntries(); $job->translationWorkflow = $command->getTranslationWorkflow(); $job->versions = $command->getVersions(); + $job->attempt = 0; $jobRecord->setAttributes($job->getAttributes(), false); diff --git a/src/services/handlers/ResendJobHandler.php b/src/services/handlers/ResendJobHandler.php new file mode 100644 index 00000000..b9310766 --- /dev/null +++ b/src/services/handlers/ResendJobHandler.php @@ -0,0 +1,67 @@ + $jobId]); + $jobRecord->status = Job::STATUS_IN_PROGRESS; + $jobRecord->liltJobId = null; + $jobRecord->save(); + + // Remove all drafts related to job + $translationRecords = TranslationRecord::findAll(['jobId' => $jobId]); + array_map(static function (TranslationRecord $t) { + if ($t->translatedDraftId !== null) { + Craft::$app->elements->deleteElementById( + $t->translatedDraftId + ); + } + }, $translationRecords); + + + // Set translation status to in progress and connector translation id to null + TranslationRecord::updateAll([ + 'status' => TranslationRecord::STATUS_IN_PROGRESS, + 'connectorTranslationId' => null, + 'translatedDraftId' => null, + 'sourceContent' => null + ], ['jobId' => $jobId]); + + // Invalidate caches for job and translation types + Craft::$app->getElements()->invalidateCachesForElementType(Job::class); + Craft::$app->getElements()->invalidateCachesForElementType(Translation::class); + + // Trigger send job to connector + Queue::push( + new SendJobToConnector(['jobId' => $jobId]), + SendJobToConnector::PRIORITY, + 10 + ); + } +} diff --git a/src/services/handlers/ResolveTranslationsConnectorIds.php b/src/services/handlers/ResolveTranslationsConnectorIds.php new file mode 100644 index 00000000..cb19f0ac --- /dev/null +++ b/src/services/handlers/ResolveTranslationsConnectorIds.php @@ -0,0 +1,153 @@ +connectorTranslationRepository + ->findByJobId($job->liltJobId); + + $connectorTranslationsMapped = $this->mapConnectorTranslations($connectorTranslations->getResults()); + + $translationResponses = $this->filterTranslationResponses($connectorTranslationsMapped); + + return $this->updateTranslationRecords($translationResponses, $job); + } + + private function mapConnectorTranslations(array $connectorTranslations): array + { + $connectorTranslationsMapped = []; + foreach ($connectorTranslations as $translationResponse) { + try { + $elementId = Craftliltplugin::getInstance() + ->connectorTranslationRepository + ->getElementIdFromTranslationResponse($translationResponse); + } catch (WrongTranslationFilenameException $ex) { + Craft::error(sprintf("Can't get element id from file: %s", $translationResponse->getName())); + continue; + } + + $targetLanguage = $this->getTranslationTargetLanguage($translationResponse); + $connectorTranslationsMapped[$elementId][$targetLanguage][] = $translationResponse; + } + + return $connectorTranslationsMapped; + } + + private function filterTranslationResponses(array $connectorTranslationsMapped): array + { + $translationResponses = []; + foreach ($connectorTranslationsMapped as $elementId => $targetLanguages) { + foreach ($targetLanguages as $targetLanguage => $translations) { + if (count($translations) === 1) { + $translationResponses[] = $translations[0]; + continue; + } + + $translationResponses[] = $this->findCompleteTranslationResponses($translations); + } + } + + return $translationResponses; + } + + /** + * @param TranslationResponse[] $translations + * @return TranslationResponse + */ + private function findCompleteTranslationResponses(array $translations): TranslationResponse + { + $result = null; + foreach ($translations as $translation) { + // Search for complete translation + if ( + $translation->getStatus() == TranslationResponse::STATUS_EXPORT_COMPLETE + || $translation->getStatus() == TranslationResponse::STATUS_MT_COMPLETE + ) { + $result = $translation; + break; + } + } + + // Fallback to first translation + if ($result === null) { + $result = $translations[0]; + } + + return $result; + } + + private function updateTranslationRecords(array $translationResponses, Job $job): array + { + $result = []; + foreach ($translationResponses as $translationResponse) { + try { + $elementId = Craftliltplugin::getInstance() + ->connectorTranslationRepository + ->getElementIdFromTranslationResponse($translationResponse); + } catch (WrongTranslationFilenameException $ex) { + Craft::error(sprintf("Can't get element id from file: %s", $translationResponse->getName())); + continue; + } + + $targetLanguage = $this->getTranslationTargetLanguage($translationResponse); + $siteId = Craftliltplugin::getInstance() + ->languageMapper + ->getSiteIdByLanguage(trim($targetLanguage, '-')); + $versionId = $job->getElementVersionId($elementId); + + $translationRecord = TranslationRecord::findOne([ + 'targetSiteId' => $siteId, + 'versionId' => $versionId, + 'jobId' => $job->id + ]); + + if ($translationRecord === null) { + throw new RuntimeException(sprintf( + "Can't find translation for target %s, jobId %d, versionId %d", + trim($targetLanguage, '-'), + $job->id, + $versionId + )); + } + + $translationRecord->connectorTranslationId = $translationResponse->getId(); + $translationRecord->save(); + + $result[$translationResponse->getId()] = $translationResponse; + } + + return array_values($result); + } + + private function getTranslationTargetLanguage(TranslationResponse $translationResponse): string + { + if (empty($translationResponse->getTrgLocale())) { + return $translationResponse->getTrgLang(); + } + + return sprintf('%s-%s', $translationResponse->getTrgLang(), $translationResponse->getTrgLocale()); + } +} diff --git a/src/services/handlers/UpdateTranslationsConnectorIds.php b/src/services/handlers/UpdateTranslationsConnectorIds.php deleted file mode 100644 index 2cb159c8..00000000 --- a/src/services/handlers/UpdateTranslationsConnectorIds.php +++ /dev/null @@ -1,75 +0,0 @@ -connectorTranslationRepository->findByJobId( - $job->liltJobId - ); - - foreach ($connectorTranslations->getResults() as $translationResponse) { - try { - $elementId = Craftliltplugin::getInstance() - ->connectorTranslationRepository - ->getElementIdFromTranslationResponse($translationResponse); - } catch (WrongTranslationFilenameException $ex) { - Craft::error( - sprintf("Can't get element id from file: %s", $translationResponse->getName()) - ); - - continue; - } - - if (empty($translationResponse->getTrgLocale())) { - $targetLanguage = $translationResponse->getTrgLang(); - } else { - $targetLanguage = sprintf( - '%s-%s', - $translationResponse->getTrgLang(), - $translationResponse->getTrgLocale() - ); - } - - $translationRecord = TranslationRecord::findOne([ - 'targetSiteId' => Craftliltplugin::getInstance() - ->languageMapper - ->getSiteIdByLanguage( - trim($targetLanguage, '-') - ), - 'versionId' => $job->getElementVersionId($elementId), - 'jobId' => $job->id - ]); - - if ($translationRecord === null) { - throw new RuntimeException( - sprintf( - "Can't find translation for target %s, jobId %d, versionId %d", - trim($targetLanguage, '-'), - $job->id, - $job->getElementVersionId($elementId) - ) - ); - } - - $translationRecord->connectorTranslationId = $translationResponse->getId(); - $translationRecord->save(); - } - } -} diff --git a/tests/integration/controllers/job/PostJobRetryControllerCest.php b/tests/integration/controllers/job/PostJobRetryControllerCest.php index a017c062..edb3d2e0 100644 --- a/tests/integration/controllers/job/PostJobRetryControllerCest.php +++ b/tests/integration/controllers/job/PostJobRetryControllerCest.php @@ -19,6 +19,7 @@ use lilthq\craftliltplugin\Craftliltplugin; use lilthq\craftliltplugin\elements\Job; use lilthq\craftliltplugin\modules\FetchJobStatusFromConnector; +use lilthq\craftliltplugin\modules\SendJobToConnector; use lilthq\craftliltplugin\parameters\CraftliltpluginParameters; use lilthq\craftliltplugin\records\JobRecord; use lilthq\craftliltplugin\records\TranslationRecord; @@ -63,45 +64,30 @@ public function testRetrySuccess(IntegrationTester $I): void [$job, $translations] = $I->createJobWithTranslations([ 'title' => 'Awesome test job', 'elementIds' => [$element->id], - 'targetSiteIds' => [Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('es-ES')], + 'targetSiteIds' => [ + Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('es-ES'), + Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('ru-RU'), + Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('de-DE'), + ], 'sourceSiteId' => Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('en-US'), 'translationWorkflow' => SettingsResponse::LILT_TRANSLATION_WORKFLOW_VERIFIED, 'versions' => [], 'authorId' => 1, ]); + foreach ($translations as $translationRecord) { + $translationRecord->status = Job::STATUS_FAILED; + $translationRecord->save(); + } + $jobRecord = JobRecord::findOne(['id' => $job->id]); $jobRecord->status = Job::STATUS_FAILED; $jobRecord->save(); - $expectQueueJob = new FetchJobStatusFromConnector([ - 'liltJobId' => 777, + $expectQueueJob = new SendJobToConnector([ 'jobId' => $job->id ]); - $I->expectJobCreateRequest( - [ - 'project_prefix' => 'Awesome test job', - 'lilt_translation_workflow' => 'VERIFIED', - ], - 200, - ['id' => 777,] - ); - - $expectedUrl = sprintf( - '/api/v1.0/jobs/777/files?name=%s' - . '&srclang=en-US' - . '&trglang=es-ES' . - '&due=', - urlencode( - sprintf('element_%d_first-entry-user-1.json+html', $element->getId()) - ) - ); - $expectedBody = ExpectedElementContent::getExpectedBody($element); - - $I->expectJobTranslationsRequest($expectedUrl, $expectedBody, HttpCode::OK); - $I->expectJobStartRequest(777, HttpCode::OK); - $I->sendAjaxPostRequest( sprintf( '?p=admin/%s', @@ -111,22 +97,16 @@ public function testRetrySuccess(IntegrationTester $I): void ); $translations = array_map(static function (TranslationRecord $translationRecord) use ($element) { - $expectedDraftBody = ExpectedElementContent::getExpectedBody( - Craft::$app->elements->getElementById( - $translationRecord->translatedDraftId, - null, - $translationRecord->targetSiteId - ) - ); - Assert::assertSame(Job::STATUS_IN_PROGRESS, $translationRecord->status); Assert::assertSame($element->id, $translationRecord->versionId); - Assert::assertEquals($expectedDraftBody, $translationRecord->sourceContent); + Assert::assertNull($translationRecord->sourceContent); + Assert::assertNull($translationRecord->translatedDraftId); + Assert::assertNull($translationRecord->connectorTranslationId); + Assert::assertSame( Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('en-US'), $translationRecord->sourceSiteId ); - Assert::assertNotNull($translationRecord->translatedDraftId); return [ 'versionId' => $translationRecord->versionId, @@ -139,12 +119,18 @@ public function testRetrySuccess(IntegrationTester $I): void ]; }, TranslationRecord::findAll(['jobId' => $job->id, 'elementId' => $element->id])); - Assert::assertCount(1, $translations); + $expectedLanguages = ['es-ES', 'ru-RU', 'de-DE']; + $actualLanguages = Craftliltplugin::getInstance()->languageMapper->getLanguagesBySiteIds( + array_column($translations, 'targetSiteId') + ); + + sort($expectedLanguages); + sort($actualLanguages); + + Assert::assertCount(3, $translations); Assert::assertEquals( - ['es-ES'], - Craftliltplugin::getInstance()->languageMapper->getLanguagesBySiteIds( - array_column($translations, 'targetSiteId') - ) + $expectedLanguages, + $actualLanguages ); $I->assertJobInQueue($expectQueueJob); diff --git a/tests/integration/modules/FetchJobStatusFromConnectorCest.php b/tests/integration/modules/FetchJobStatusFromConnectorCest.php index 950b663b..a013d850 100644 --- a/tests/integration/modules/FetchJobStatusFromConnectorCest.php +++ b/tests/integration/modules/FetchJobStatusFromConnectorCest.php @@ -12,9 +12,14 @@ use IntegrationTester; use LiltConnectorSDK\Model\JobResponse; use LiltConnectorSDK\Model\SettingsResponse; +use LiltConnectorSDK\Model\TranslationResponse; use lilthq\craftliltplugin\Craftliltplugin; +use lilthq\craftliltplugin\elements\Job; +use lilthq\craftliltplugin\elements\Translation; use lilthq\craftliltplugin\modules\FetchJobStatusFromConnector; use lilthq\craftliltplugin\modules\FetchTranslationFromConnector; +use lilthq\craftliltplugin\modules\SendJobToConnector; +use lilthq\craftliltplugin\records\JobRecord; use lilthq\craftliltplugin\records\TranslationRecord; use lilthq\craftliltplugin\services\repositories\SettingsRepository; use lilthq\craftliltplugintests\integration\AbstractIntegrationCest; @@ -79,7 +84,7 @@ public function testExecuteSuccessVerified(IntegrationTester $I): void 'errorMsg' => null, 'id' => 11111, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'export_complete', + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, 'trgLang' => 'es', 'trgLocale' => 'ES', 'updatedAt' => '2022-06-02T23:01:42', @@ -89,7 +94,7 @@ public function testExecuteSuccessVerified(IntegrationTester $I): void 'errorMsg' => null, 'id' => 22222, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'export_complete', + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, 'trgLang' => 'de', 'trgLocale' => 'DE', 'updatedAt' => '2022-06-02T23:01:42', @@ -99,7 +104,7 @@ public function testExecuteSuccessVerified(IntegrationTester $I): void 'errorMsg' => null, 'id' => 33333, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'export_complete', + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, 'trgLang' => 'ru', 'trgLocale' => 'RU', 'updatedAt' => '2022-06-02T23:01:42', @@ -167,6 +172,170 @@ public function testExecuteSuccessVerified(IntegrationTester $I): void } } + /** + * @throws Exception + * @throws ModuleException + */ + public function testExecuteSuccessVerifiedDuplicate(IntegrationTester $I): void + { + $I->clearQueue(); + $I->disableOption(SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC); + + Db::truncateTable(Craft::$app->queue->tableName); + + $user = Craft::$app->getUsers()->getUserById(1); + $I->amLoggedInAs($user); + + $element = Entry::find() + ->where(['authorId' => 1]) + ->orderBy(['id' => SORT_DESC]) + ->one(); + + [$job, $translations] = $I->createJobWithTranslations([ + 'title' => 'Awesome test job', + 'elementIds' => [$element->id], + 'targetSiteIds' => '*', + 'sourceSiteId' => Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('en-US'), + 'translationWorkflow' => SettingsResponse::LILT_TRANSLATION_WORKFLOW_VERIFIED, + 'versions' => [], + 'authorId' => 1, + 'liltJobId' => 777, + ]); + + $I->expectJobGetRequest( + 777, + 200, + [ + 'status' => JobResponse::STATUS_COMPLETE + ] + ); + + $responseBody = [ + 'limit' => 25, + 'results' => [ + 0 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 11111, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, + 'trgLang' => 'es', + 'trgLocale' => 'ES', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 1 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 22222, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, + 'trgLang' => 'de', + 'trgLocale' => 'DE', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 2 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 33333, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_IMPORT_COMPLETE, + 'trgLang' => 'ru', + 'trgLocale' => 'RU', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 3 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 44444, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_IMPORT_COMPLETE, + 'trgLang' => 'ru', + 'trgLocale' => 'RU', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 4 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 55555, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_EXPORT_COMPLETE, + 'trgLang' => 'ru', + 'trgLocale' => 'RU', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 5 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 66666, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_IMPORT_COMPLETE, + 'trgLang' => 'ru', + 'trgLocale' => 'RU', + 'updatedAt' => '2022-06-02T23:01:42', + ], + ], + 'start' => 0, + ]; + + $I->expectTranslationsGetRequest( + 777, + 0, + 100, + HttpCode::OK, + $responseBody + ); + + $I->runQueue( + FetchJobStatusFromConnector::class, + [ + 'liltJobId' => $job->liltJobId, + 'jobId' => $job->id, + ] + ); + + $totalJobs = Craft::$app->queue->getJobInfo(); + + Assert::assertCount(3, $totalJobs); + $I->assertJobInQueue( + new FetchTranslationFromConnector([ + 'jobId' => $job->id, + 'translationId' => $translations[0]->id, + 'liltJobId' => 777 + ]) + ); + + $I->assertJobInQueue( + new FetchTranslationFromConnector([ + 'jobId' => $job->id, + 'translationId' => $translations[1]->id, + 'liltJobId' => 777 + ]) + ); + + $I->assertJobInQueue( + new FetchTranslationFromConnector([ + 'jobId' => $job->id, + 'translationId' => $translations[2]->id, + 'liltJobId' => 777 + ]) + ); + + $translationAssertions = [ + 'es-ES' => 11111, + 'de-DE' => 22222, + 'ru-RU' => 55555, + ]; + foreach ($translationAssertions as $language => $expectedConnectorTranslationId) { + $translationEs = TranslationRecord::findOne([ + 'jobId' => $job->id, + 'elementId' => $element->id, + 'targetSiteId' => Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage($language) + ]); + + Assert::assertSame($expectedConnectorTranslationId, $translationEs->connectorTranslationId); + } + } + /** * @throws Exception * @throws ModuleException @@ -213,7 +382,7 @@ public function testExecuteSuccessInstant(IntegrationTester $I): void 'errorMsg' => null, 'id' => 11111, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'mt_complete', + 'status' => TranslationResponse::STATUS_MT_COMPLETE, 'trgLang' => 'es', 'trgLocale' => 'ES', 'updatedAt' => '2022-06-02T23:01:42', @@ -223,7 +392,7 @@ public function testExecuteSuccessInstant(IntegrationTester $I): void 'errorMsg' => null, 'id' => 22222, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'mt_complete', + 'status' => TranslationResponse::STATUS_MT_COMPLETE, 'trgLang' => 'de', 'trgLocale' => 'DE', 'updatedAt' => '2022-06-02T23:01:42', @@ -233,7 +402,7 @@ public function testExecuteSuccessInstant(IntegrationTester $I): void 'errorMsg' => null, 'id' => 33333, 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), - 'status' => 'mt_complete', + 'status' => TranslationResponse::STATUS_MT_COMPLETE, 'trgLang' => 'ru', 'trgLocale' => 'RU', 'updatedAt' => '2022-06-02T23:01:42', @@ -301,6 +470,132 @@ public function testExecuteSuccessInstant(IntegrationTester $I): void } } + /** + * @throws Exception + * @throws ModuleException + */ + public function testExecuteSuccessInstantRetry(IntegrationTester $I): void + { + $I->clearQueue(); + $I->disableOption(SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC); + + Db::truncateTable(Craft::$app->queue->tableName); + + $user = Craft::$app->getUsers()->getUserById(1); + $I->amLoggedInAs($user); + + $element = Entry::find() + ->where(['authorId' => 1]) + ->orderBy(['id' => SORT_DESC]) + ->one(); + + [$job, $translations] = $I->createJobWithTranslations([ + 'title' => 'Awesome test job', + 'elementIds' => [$element->id], + 'targetSiteIds' => '*', + 'sourceSiteId' => Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage('en-US'), + 'translationWorkflow' => SettingsResponse::LILT_TRANSLATION_WORKFLOW_INSTANT, + 'versions' => [], + 'authorId' => 1, + 'liltJobId' => 777, + ]); + + $I->expectJobGetRequest( + 777, + 200, + [ + 'status' => JobResponse::STATUS_FAILED + ] + ); + + $responseBody = [ + 'limit' => 25, + 'results' => [ + 0 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 11111, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_MT_COMPLETE, + 'trgLang' => 'es', + 'trgLocale' => 'ES', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 1 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 22222, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_MT_COMPLETE, + 'trgLang' => 'de', + 'trgLocale' => 'DE', + 'updatedAt' => '2022-06-02T23:01:42', + ], + 2 => [ + 'createdAt' => '2022-05-29T11:31:58', + 'errorMsg' => null, + 'id' => 33333, + 'name' => sprintf('497058_element_%d_first-entry-user-1.json+html', $element->id), + 'status' => TranslationResponse::STATUS_MT_COMPLETE, + 'trgLang' => 'ru', + 'trgLocale' => 'RU', + 'updatedAt' => '2022-06-02T23:01:42', + ], + ], + 'start' => 0, + ]; + + $I->expectTranslationsGetRequest( + 777, + 0, + 100, + HttpCode::OK, + $responseBody + ); + + $I->runQueue( + FetchJobStatusFromConnector::class, + [ + 'liltJobId' => $job->liltJobId, + 'jobId' => $job->id, + ] + ); + + $totalJobs = Craft::$app->queue->getJobInfo(); + + Assert::assertCount(1, $totalJobs); + $I->assertJobInQueue( + new SendJobToConnector([ + 'jobId' => $job->id, + ]) + ); + + $jobRecord = JobRecord::findOne(['id' => $job->id]); + Assert::assertSame(Job::STATUS_IN_PROGRESS, $jobRecord->status); + Assert::assertNull($jobRecord->liltJobId); + Assert::assertSame(1, $jobRecord->attempt); + + $translationAssertions = [ + 'es-ES' => null, + 'de-DE' => null, + 'ru-RU' => null, + ]; + foreach ($translationAssertions as $language => $expectedConnectorTranslationId) { + $actualTranslation = TranslationRecord::findOne([ + 'jobId' => $job->id, + 'elementId' => $element->id, + 'targetSiteId' => Craftliltplugin::getInstance()->languageMapper->getSiteIdByLanguage($language) + ]); + + Assert::assertSame($expectedConnectorTranslationId, $actualTranslation->connectorTranslationId); + Assert::assertSame(TranslationRecord::STATUS_IN_PROGRESS, $actualTranslation->status); + + Assert::assertNull($actualTranslation->connectorTranslationId); + Assert::assertNull($actualTranslation->translatedDraftId); + Assert::assertNull($actualTranslation->sourceContent); + } + } + /** * @throws Exception * @throws ModuleException