Merge pull request #153 from lilt/3.x-attempts

lilt · May 28, 2024 · a8b32e5 · a8b32e5
2 parents 7ca4c01 + f482d6a
commit a8b32e5
Show file tree

Hide file tree

Showing 18 changed files with 678 additions and 230 deletions.
diff --git a/Makefile b/Makefile
@@ -7,86 +7,89 @@ PHP_VERSION?=8.1
 MYSQL_VERSION?=5.7
 
 up:
-	docker-compose up -d
-	docker-compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done'
+	docker compose up -d
+	docker compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done'
+
+create-migration:
+	docker compose exec -T -u www-data cli-app sh -c "php craft migrate/create --plugin=craft-lilt-plugin add_job_attempts"
 
 down:
-	docker-compose down -v --remove-orphans
+	docker compose down -v --remove-orphans
 
 restart: down
-	docker-compose up -d
+	docker compose up -d
 
 cli:
-	docker-compose exec -u www-data cli-app sh
+	docker compose exec -u www-data cli-app sh
 
 root:
-	docker-compose exec -u root cli-app sh
+	docker compose exec -u root cli-app sh
 
 composer-install:
-	docker-compose exec -T -u root cli-app sh -c "apk add git"
-	docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
-	docker-compose exec -T -u root cli-app sh -c "rm -f composer.lock"
-	docker-compose exec -T -u root cli-app sh -c "rm -rf vendor"
-	docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
-	docker-compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php"
-	docker-compose exec -T -u www-data cli-app sh -c "php composer.phar install"
+	docker compose exec -T -u root cli-app sh -c "apk add git"
+	docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
+	docker compose exec -T -u root cli-app sh -c "rm -f composer.lock"
+	docker compose exec -T -u root cli-app sh -c "rm -rf vendor"
+	docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
+	docker compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php"
+	docker compose exec -T -u www-data cli-app sh -c "php composer.phar install"
 
 quality:
-	docker-compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar"
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs"
-	docker-compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations"
+	docker compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs"
+	docker compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations"
 
 quality-fix:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf"
 
 codecept-build:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build"
 
 coverage-xdebug:
-	docker-compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
+	docker compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
 
 install-pcov:
-	docker-compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c"
-	docker-compose exec -T -u root cli-app sh -c "pecl install pcov || true"
-	docker-compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov"
+	docker compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c"
+	docker compose exec -T -u root cli-app sh -c "pecl install pcov || true"
+	docker compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov"
 
 coverage: install-pcov
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
 
 tests-with-coverage: codecept-build install-pcov unit-coverage integration-coverage functional-coverage
 
 integration-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml"
 
 functional-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml"
 
 unit-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml"
 
 integration: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration"
 
 functional: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional"
 
 unit: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit"
 
 test: functional integration unit
 
 prepare-container:
-	PHP_VERSION=7.2 docker-compose up -d
-	docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
-	docker-compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git"
-	docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
-	docker-compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php"
-	docker-compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer"
+	PHP_VERSION=7.2 docker compose up -d
+	docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
+	docker compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git"
+	docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
+	docker compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php"
+	docker compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer"
 
 test-craft-versions: prepare-container
-	docker-compose exec -T -u www-data cli-app bash -c \
+	docker compose exec -T -u www-data cli-app bash -c \
 		"./craft-versions.sh ${CRAFT_VERSION}"
 
 require-guzzle-v6:
-	docker-compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true"
-	docker-compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi'
+	docker compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true"
+	docker compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi'
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   cli-app:
     image: craftcms/cli:${PHP_VERSION}

diff --git a/src/Craftliltplugin.php b/src/Craftliltplugin.php
@@ -36,13 +36,14 @@
 use lilthq\craftliltplugin\services\handlers\LoadI18NHandler;
 use lilthq\craftliltplugin\services\handlers\PublishDraftHandler;
 use lilthq\craftliltplugin\services\handlers\RefreshJobStatusHandler;
+use lilthq\craftliltplugin\services\handlers\ResendJobHandler;
 use lilthq\craftliltplugin\services\handlers\SendJobToLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\SendTranslationToLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\StartQueueManagerHandler;
 use lilthq\craftliltplugin\services\handlers\SyncJobFromLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\TranslationFailedHandler;
 use lilthq\craftliltplugin\services\handlers\UpdateJobStatusHandler;
-use lilthq\craftliltplugin\services\handlers\UpdateTranslationsConnectorIds;
+use lilthq\craftliltplugin\services\handlers\ResolveTranslationsConnectorIds;
 use lilthq\craftliltplugin\services\listeners\ListenerRegister;
 use lilthq\craftliltplugin\services\mappers\LanguageMapper;
 use lilthq\craftliltplugin\services\providers\ConnectorConfigurationProvider;
@@ -109,8 +110,9 @@
  * @property CreateDraftHandler $createDraftHandler
  * @property CopySourceTextHandler $copySourceTextHandler
  * @property UpdateJobStatusHandler $updateJobStatusHandler
+ * @property ResendJobHandler $resendJobHandler
  * @property SettingsRepository $settingsRepository
- * @property UpdateTranslationsConnectorIds $updateTranslationsConnectorIds
+ * @property ResolveTranslationsConnectorIds $resolveTranslationsConnectorIds
  * @property PackagistRepository $packagistRepository
  * @property StartQueueManagerHandler $startQueueManagerHandler
  * @property ServiceInitializer $serviceInitializer

diff --git a/src/controllers/job/PostJobRetryController.php b/src/controllers/job/PostJobRetryController.php
@@ -46,7 +46,7 @@ public function actionInvoke(): Response
                 sprintf('Job retried (previous Lilt Job ID: %d)', $job->liltJobId)
             );
 
-            Craftliltplugin::getInstance()->sendJobToLiltConnectorHandler->__invoke($job);
+            Craftliltplugin::getInstance()->resendJobHandler->__invoke($job->id);
         }
 
 

diff --git a/src/elements/Job.php b/src/elements/Job.php
@@ -43,6 +43,7 @@ class Job extends Element
     public const STATUS_FAILED = 'failed';
     public const STATUS_NEEDS_ATTENTION = 'needs-attention';
 
+    public const MAX_JOB_ATTEMPTS = 5;
 
     public $uid;
     public $authorId;
@@ -59,6 +60,9 @@ class Job extends Element
     public $dateCreated;
     public $dateUpdated;
 
+    public $attempt = 0;
+
+
     // @codingStandardsIgnoreStart
     private $_author;
     private $_elements;

diff --git a/src/migrations/Install.php b/src/migrations/Install.php
@@ -32,6 +32,7 @@ public function safeUp(): void
 
         $this->createTable(CraftliltpluginParameters::JOB_TABLE_NAME, [
             'id' => $this->primaryKey()->unsigned(),
+            'attempt' => $this->integer()->notNull()->defaultValue(0),
             'title' => $this->string()->null(),
             'authorId' => $this->integer()->null(),
             'liltJobId' => $this->integer()->null(),

diff --git a/src/migrations/m240526_212007_add_job_attempts.php b/src/migrations/m240526_212007_add_job_attempts.php
@@ -0,0 +1,39 @@
+<?php
+
+namespace lilthq\craftliltplugin\migrations;
+
+use craft\db\Migration;
+use lilthq\craftliltplugin\parameters\CraftliltpluginParameters;
+
+/**
+ * m240526_212007_add_job_attempts migration.
+ */
+// @codingStandardsIgnoreStart
+class m240526_212007_add_job_attempts extends Migration
+// @codingStandardsIgnoreEnd
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp()
+    {
+        // Add the 'attempt' column to the 'lilt_jobs' table
+        $this->addColumn(
+            CraftliltpluginParameters::JOB_TABLE_NAME,
+            'attempt',
+            $this->integer()->unsigned()->notNull()->defaultValue(0)
+        );
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown()
+    {
+        // Remove the 'attempt' column from the 'lilt_jobs' table
+        $this->dropColumn(
+            CraftliltpluginParameters::JOB_TABLE_NAME,
+            'attempt'
+        );
+    }
+}
diff --git a/src/modules/FetchJobStatusFromConnector.php b/src/modules/FetchJobStatusFromConnector.php
@@ -79,14 +79,39 @@ public function execute($queue): void
             || $liltJob->getStatus() === JobResponse::STATUS_FAILED;
 
         if ($isJobFailed) {
-            $jobRecord->status = Job::STATUS_FAILED;
-
             Craftliltplugin::getInstance()->jobLogsRepository->create(
                 $jobRecord->id,
                 Craft::$app->getUser()->getId(),
                 sprintf('Job failed, received status: %s', $liltJob->getStatus())
             );
 
+            if ($jobRecord->attempt < Job::MAX_JOB_ATTEMPTS) {
+                // Retry job
+                $jobRecord->attempt++;
+                $jobRecord->save();
+
+                Craftliltplugin::getInstance()->resendJobHandler->__invoke(
+                    $jobRecord->id
+                );
+
+
+                Craftliltplugin::getInstance()->jobLogsRepository->create(
+                    $jobRecord->id,
+                    Craft::$app->getUser()->getId(),
+                    sprintf(
+                        'Trying again to send job, attempt: %d/%d',
+                        $jobRecord->attempt,
+                        Job::MAX_JOB_ATTEMPTS
+                    )
+                );
+
+                $mutex->release($mutexKey);
+                $this->markAsDone($queue);
+                return;
+            }
+
+            // Job failed, set status to failed
+            $jobRecord->status = Job::STATUS_FAILED;
             $jobRecord->save();
 
             TranslationRecord::updateAll(
@@ -108,7 +133,7 @@ public function execute($queue): void
         }
 
         if (!$isJobFinished) {
-            $queueDisableAutomaticSync = (bool) Craftliltplugin::getInstance()->settingsRepository->get(
+            $queueDisableAutomaticSync = (bool)Craftliltplugin::getInstance()->settingsRepository->get(
                 SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC
             );
 
@@ -132,15 +157,12 @@ public function execute($queue): void
             return;
         }
 
-        $connectorTranslations = Craftliltplugin::getInstance()->connectorTranslationRepository->findByJobId(
-            $job->liltJobId
-        );
-
+        $connectorTranslations = Craftliltplugin::getInstance()->resolveTranslationsConnectorIds->update($job);
         $connectorTranslationsStatuses = array_map(
             function (TranslationResponse $connectorTranslation) {
                 return $connectorTranslation->getStatus();
             },
-            $connectorTranslations->getResults()
+            $connectorTranslations
         );
 
         $translationFinished =
@@ -184,7 +206,7 @@ function (TranslationResponse $connectorTranslation) {
                 return;
             }
 
-            $queueDisableAutomaticSync = (bool) Craftliltplugin::getInstance()->settingsRepository->get(
+            $queueDisableAutomaticSync = (bool)Craftliltplugin::getInstance()->settingsRepository->get(
                 SettingsRepository::QUEUE_DISABLE_AUTOMATIC_SYNC
             );
 
@@ -208,7 +230,7 @@ function (TranslationResponse $connectorTranslation) {
             return;
         }
 
-        if ($jobRecord->isVerifiedFlow()) {
+        if ($jobRecord->isVerifiedFlow() || $jobRecord->isInstantFlow()) {
             #LILT_TRANSLATION_WORKFLOW_VERIFIED
 
             $jobRecord->status = Job::STATUS_IN_PROGRESS;
@@ -217,58 +239,6 @@ function (TranslationResponse $connectorTranslation) {
                 $this->jobId
             );
 
-            Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job);
-
-            foreach ($translations as $translation) {
-                Queue::push(
-                    new FetchTranslationFromConnector(
-                        [
-                            'jobId' => $this->jobId,
-                            'liltJobId' => $this->liltJobId,
-                            'translationId' => $translation->id,
-                        ]
-                    ),
-                    FetchTranslationFromConnector::PRIORITY,
-                    10, //10 seconds for first job
-                    FetchTranslationFromConnector::TTR
-                );
-            }
-        }
-
-        if ($jobRecord->isInstantFlow()) {
-            #LILT_TRANSLATION_WORKFLOW_INSTANT
-
-            if (
-                $liltJob->getStatus() === JobResponse::STATUS_FAILED
-                || $liltJob->getStatus() === JobResponse::STATUS_CANCELED
-            ) {
-                $jobRecord->status = Job::STATUS_FAILED;
-
-                TranslationRecord::updateAll(
-                    ['status' => TranslationRecord::STATUS_FAILED],
-                    ['jobId' => $jobRecord->id]
-                );
-
-                Craft::error([
-                    "message" => sprintf(
-                        'Set job %d and translations to status failed due to failed/cancel status from lilt',
-                        $jobRecord->id
-                    ),
-                    "jobRecord" => $jobRecord,
-                ]);
-
-                $mutex->release($mutexKey);
-                $this->markAsDone($queue);
-
-                return;
-            }
-
-            $translations = Craftliltplugin::getInstance()->translationRepository->findByJobId(
-                $this->jobId
-            );
-
-            Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job);
-
             foreach ($translations as $translation) {
                 Queue::push(
                     new FetchTranslationFromConnector(

diff --git a/src/modules/FetchTranslationFromConnector.php b/src/modules/FetchTranslationFromConnector.php
@@ -76,7 +76,7 @@ public function execute($queue): void
         }
 
         if (empty($translationRecord->connectorTranslationId)) {
-            Craftliltplugin::getInstance()->updateTranslationsConnectorIds->update($job);
+            Craftliltplugin::getInstance()->resolveTranslationsConnectorIds->update($job);
         }
         $translationRecord->refresh();
 

diff --git a/src/records/JobRecord.php b/src/records/JobRecord.php
@@ -26,6 +26,7 @@
  * @property int $sourceSiteLanguage [int(11) unsigned]
  * @property string $targetSiteIds [json]
  * @property string $dueDate [datetime]
+ * @property int $attempt [int(11) unsigned]
  *
  * @property-read ActiveQueryInterface $element
  * @property string $translationWorkflow [varchar(50)]
-Original file line number
+Diff line change
@@ Expand Up / @@ -46,7 +46,7 @@ public function actionInvoke(): Response @@
                     sprintf('Job retried (previous Lilt Job ID: %d)', $job->liltJobId)
                 );
-                Craftliltplugin::getInstance()->sendJobToLiltConnectorHandler->__invoke($job);
+                Craftliltplugin::getInstance()->resendJobHandler->__invoke($job->id);
             }
@@ Expand Down @@