Improve retry logic and logs

Makefile

@@ -7,86 +7,89 @@ PHP_VERSION?=8.1
 MYSQL_VERSION?=5.7
 
 up:
-	docker-compose up -d
-	docker-compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done'
+	docker compose up -d
+	docker compose exec -T mysql-test sh -c 'while ! mysqladmin ping -h"mysql-test" --silent; do sleep 1; done'
+
+create-migration:
+	docker compose exec -T -u www-data cli-app sh -c "php craft migrate/create --plugin=craft-lilt-plugin add_job_attempts"
 
 down:
-	docker-compose down -v --remove-orphans
+	docker compose down -v --remove-orphans
 
 restart: down
-	docker-compose up -d
+	docker compose up -d
 
 cli:
-	docker-compose exec -u www-data cli-app sh
+	docker compose exec -u www-data cli-app sh
 
 root:
-	docker-compose exec -u root cli-app sh
+	docker compose exec -u root cli-app sh
 
 composer-install:
-	docker-compose exec -T -u root cli-app sh -c "apk add git"
-	docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
-	docker-compose exec -T -u root cli-app sh -c "rm -f composer.lock"
-	docker-compose exec -T -u root cli-app sh -c "rm -rf vendor"
-	docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
-	docker-compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php"
-	docker-compose exec -T -u www-data cli-app sh -c "php composer.phar install"
+	docker compose exec -T -u root cli-app sh -c "apk add git"
+	docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
+	docker compose exec -T -u root cli-app sh -c "rm -f composer.lock"
+	docker compose exec -T -u root cli-app sh -c "rm -rf vendor"
+	docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
+	docker compose exec -T -u www-data cli-app sh -c "curl -s https://getcomposer.org/installer | php"
+	docker compose exec -T -u www-data cli-app sh -c "php composer.phar install"
 
 quality:
-	docker-compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar"
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs"
-	docker-compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations"
+	docker compose exec -T -u www-data cli-app sh -c "curl -L -s https://phar.phpunit.de/phpcpd.phar --output phpcpd.phar"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcs"
+	docker compose exec -T -u www-data cli-app sh -c "php phpcpd.phar src --exclude /craft-lilt-plugin/src/migrations"
 
 quality-fix:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/phpcbf"
 
 codecept-build:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept build"
 
 coverage-xdebug:
-	docker-compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
+	docker compose exec -T -u www-data cli-app sh -c "php -dxdebug.mode=coverage vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
 
 install-pcov:
-	docker-compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c"
-	docker-compose exec -T -u root cli-app sh -c "pecl install pcov || true"
-	docker-compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov"
+	docker compose exec -T -u root cli-app sh -c "apk --no-cache add pcre-dev autoconf dpkg-dev dpkg file g++ gcc libc-dev make pkgconf re2c"
+	docker compose exec -T -u root cli-app sh -c "pecl install pcov || true"
+	docker compose exec -T -u root cli-app sh -c "docker-php-ext-enable pcov"
 
 coverage: install-pcov
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run --coverage --coverage-xml --coverage-html"
 
 tests-with-coverage: codecept-build install-pcov unit-coverage integration-coverage functional-coverage
 
 integration-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration --coverage-xml=coverage-integration.xml"
 
 functional-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional --coverage-xml=coverage-functional.xml"
 
 unit-coverage:
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit --coverage-xml=coverage-unit.xml"
 
 integration: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run integration"
 
 functional: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run functional"
 
 unit: codecept-build
-	docker-compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit"
+	docker compose exec -T -u www-data cli-app sh -c "php vendor/bin/codecept run unit"
 
 test: functional integration unit
 
 prepare-container:
-	PHP_VERSION=7.2 docker-compose up -d
-	docker-compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
-	docker-compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git"
-	docker-compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
-	docker-compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php"
-	docker-compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer"
+	PHP_VERSION=7.2 docker compose up -d
+	docker compose exec -T -u root cli-app sh -c "chown -R www-data:www-data /craft-lilt-plugin"
+	docker compose exec -T -u root cli-app sh -c "apk --no-cache add bash make git"
+	docker compose exec -T -u www-data cli-app sh -c "cp tests/.env.test tests/.env"
+	docker compose exec -T -u root cli-app sh -c "curl -s https://getcomposer.org/installer | php"
+	docker compose exec -T -u root cli-app sh -c "cp composer.phar /bin/composer"
 
 test-craft-versions: prepare-container
-	docker-compose exec -T -u www-data cli-app bash -c \
+	docker compose exec -T -u www-data cli-app bash -c \
 		"./craft-versions.sh ${CRAFT_VERSION}"
 
 require-guzzle-v6:
-	docker-compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true"
-	docker-compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi'
+	docker compose exec -T -u www-data cli-app sh -c "php composer.phar require guzzlehttp/guzzle:^6.0 -W --no-scripts || true"
+	docker compose exec -T -u www-data cli-app sh -c 'if ! php composer.phar show -i | grep "guzzlehttp/guzzle" | grep "6."; then echo "Guzzle version 6 is not present."; exit 1; fi'

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   cli-app:
     image: craftcms/cli:${PHP_VERSION}

e2e/Makefile

@@ -54,7 +54,7 @@ up: clone down
 	docker-compose exec -T app sh -c 'php craft project-config/rebuild'
 	docker-compose exec -T app sh -c 'php craft up'
 	docker-compose exec -T app sh -c 'php craft migrate/up'
-	docker-compose exec -T app sh -c 'php craft queue/run'
+	docker-compose exec -T app sh -c 'nohup ./queue_listen.sh > queue.log 2>&1 &'
 
 cli:
 	docker-compose exec -T app sh

e2e/docker-compose.yml

@@ -28,12 +28,19 @@ services:
       MOCKSERVER_CORS_ALLOW_METHODS: "CONNECT, DELETE, GET, HEAD, OPTIONS, POST, PUT, PATCH, TRACE"
     networks:
       - e2e-network
+
+  mockserver-healthcheck:
+    image: alpine:latest
     healthcheck:
-      test: [ "CMD-SHELL", "curl --fail http://localhost:1080/status" ]
+      test: ["CMD-SHELL", "curl --fail -XPUT http://mockserver:1080/status || exit 1"]
       interval: 5s
       timeout: 5s
       retries: 3
-    restart: on-failure
+    depends_on:
+      - mockserver
+    networks:
+      - e2e-network
+    entrypoint: ["sh", "-c", "apk add --no-cache curl && while sleep 3600; do :; done"]
 
 networks:
   e2e-network:

e2e/happy-lager-override/config/general.php

@@ -38,6 +38,7 @@
 
     // Whether crawlers should be allowed to index pages and following links
     'disallowRobots' => !$isProd,
+    'runQueueAutomatically' => false,
 
     'aliases' => [
         '@assetBasePath' => App::env('ASSET_BASE_PATH') ?: "./assets",

e2e/happy-lager-override/queue_listen.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+while true; do
+    echo "Starting the queue listen process..."
+    php craft queue/listen -v
+    echo "Queue listen completed. Restarting in 30 seconds..."
+    sleep 1
+done

src/Craftliltplugin.php

@@ -36,13 +36,14 @@
 use lilthq\craftliltplugin\services\handlers\LoadI18NHandler;
 use lilthq\craftliltplugin\services\handlers\PublishDraftHandler;
 use lilthq\craftliltplugin\services\handlers\RefreshJobStatusHandler;
+use lilthq\craftliltplugin\services\handlers\ResendJobHandler;
 use lilthq\craftliltplugin\services\handlers\SendJobToLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\SendTranslationToLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\StartQueueManagerHandler;
 use lilthq\craftliltplugin\services\handlers\SyncJobFromLiltConnectorHandler;
 use lilthq\craftliltplugin\services\handlers\TranslationFailedHandler;
 use lilthq\craftliltplugin\services\handlers\UpdateJobStatusHandler;
-use lilthq\craftliltplugin\services\handlers\UpdateTranslationsConnectorIds;
+use lilthq\craftliltplugin\services\handlers\ResolveTranslationsConnectorIds;
 use lilthq\craftliltplugin\services\listeners\ListenerRegister;
 use lilthq\craftliltplugin\services\mappers\LanguageMapper;
 use lilthq\craftliltplugin\services\providers\ConnectorConfigurationProvider;
@@ -109,8 +110,9 @@
  * @property CreateDraftHandler $createDraftHandler
  * @property CopySourceTextHandler $copySourceTextHandler
  * @property UpdateJobStatusHandler $updateJobStatusHandler
+ * @property ResendJobHandler $resendJobHandler
  * @property SettingsRepository $settingsRepository
- * @property UpdateTranslationsConnectorIds $updateTranslationsConnectorIds
+ * @property ResolveTranslationsConnectorIds $resolveTranslationsConnectorIds
  * @property PackagistRepository $packagistRepository
  * @property StartQueueManagerHandler $startQueueManagerHandler
  * @property ServiceInitializer $serviceInitializer

src/controllers/job/PostJobRetryController.php

@@ -5,12 +5,24 @@
 namespace lilthq\craftliltplugin\controllers\job;
 
 use Craft;
+use craft\errors\ElementNotFoundException;
 use craft\web\Controller;
+use LiltConnectorSDK\ApiException;
 use lilthq\craftliltplugin\Craftliltplugin;
+use Throwable;
+use yii\base\Exception;
+use yii\db\StaleObjectException;
 use yii\web\Response;
 
 class PostJobRetryController extends Controller
 {
+    /**
+     * @throws ElementNotFoundException
+     * @throws Throwable
+     * @throws ApiException
+     * @throws StaleObjectException
+     * @throws Exception
+     */
     public function actionInvoke(): Response
     {
         $request = Craft::$app->getRequest();
@@ -34,7 +46,7 @@ public function actionInvoke(): Response
                 sprintf('Job retried (previous Lilt Job ID: %d)', $job->liltJobId)
             );
 
-            Craftliltplugin::getInstance()->sendJobToLiltConnectorHandler->__invoke($job);
+            Craftliltplugin::getInstance()->resendJobHandler->__invoke($job->id);
         }
 
 

src/elements/Job.php

@@ -43,6 +43,7 @@ class Job extends Element
     public const STATUS_FAILED = 'failed';
     public const STATUS_NEEDS_ATTENTION = 'needs-attention';
 
+    public const MAX_JOB_ATTEMPTS = 5;
 
     public $uid;
     public $authorId;
@@ -59,6 +60,9 @@ class Job extends Element
     public $dateCreated;
     public $dateUpdated;
 
+    public $attempt = 0;
+
+
     // @codingStandardsIgnoreStart
     private $_author;
     private $_elements;

src/migrations/Install.php

@@ -32,6 +32,7 @@ public function safeUp(): void
 
         $this->createTable(CraftliltpluginParameters::JOB_TABLE_NAME, [
             'id' => $this->primaryKey()->unsigned(),
+            'attempt' => $this->integer()->notNull()->defaultValue(0),
             'title' => $this->string()->null(),
             'authorId' => $this->integer()->null(),
             'liltJobId' => $this->integer()->null(),

src/migrations/m240526_212007_add_job_attempts.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace lilthq\craftliltplugin\migrations;
+
+use craft\db\Migration;
+use lilthq\craftliltplugin\parameters\CraftliltpluginParameters;
+
+/**
+ * m240526_212007_add_job_attempts migration.
+ */
+// @codingStandardsIgnoreStart
+class m240526_212007_add_job_attempts extends Migration
+// @codingStandardsIgnoreEnd
+{
+    /**
+     * @inheritdoc
+     */
+    public function safeUp()
+    {
+        // Add the 'attempt' column to the 'lilt_jobs' table
+        $this->addColumn(
+            CraftliltpluginParameters::JOB_TABLE_NAME,
+            'attempt',
+            $this->integer()->unsigned()->notNull()->defaultValue(0)
+        );
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function safeDown()
+    {
+        // Remove the 'attempt' column from the 'lilt_jobs' table
+        $this->dropColumn(
+            CraftliltpluginParameters::JOB_TABLE_NAME,
+            'attempt'
+        );
+    }
+}

src/modules/AbstractRetryJob.php

@@ -37,6 +37,9 @@ abstract class AbstractRetryJob extends BaseJob
      */
     abstract public function canRetry(): bool;
 
+    /**
+     * @return SendJobToConnector|SendTranslationToConnector|FetchJobStatusFromConnector|FetchTranslationFromConnector
+     */
     abstract public function getRetryJob(): BaseJob;
 
     protected function getCommand(): ?Command