Rework of module updates : Introducing Sources and their Providers

classes/Parameters/UpgradeFileNames.php

@@ -136,6 +136,18 @@ class UpgradeFileNames
      */
     const TRANSLATION_FILES_CUSTOM_LIST = 'translations-custom.list';
 
+    /**
+     * Module Sources Providers are classes that fetching & compute data
+     * from the filesystem or an external resource like an API.
+     * Caching the data avoids this computation to happen again before each module.
+     *
+     * @var string
+     */
+    const MODULE_SOURCE_PROVIDER_CACHE_LOCAL = 'moduleSourcesLocal.cache';
+    const MODULE_SOURCE_PROVIDER_CACHE_COMPOSER = 'moduleSourcesComposer.cache';
+    const MODULE_SOURCE_PROVIDER_CACHE_MARKETPLACE_API = 'moduleSourcesMarketplace.cache';
+    const MODULE_SOURCE_PROVIDER_CACHE_DISTRIBUTION_API = 'moduleSourcesDistribution.cache';
+
     /**
      * tmp_files contains an array of filename which will be removed
      * at the beginning of the upgrade process.
@@ -155,5 +167,9 @@ class UpgradeFileNames
         'MAILS_CUSTOM_LIST',
         'TRANSLATION_FILES_CUSTOM_LIST',
         'MODULES_TO_UPGRADE_LIST',
+        'MODULE_SOURCE_PROVIDER_CACHE_LOCAL',
+        'MODULE_SOURCE_PROVIDER_CACHE_COMPOSER',
+        'MODULE_SOURCE_PROVIDER_CACHE_MARKETPLACE_API',
+        'MODULE_SOURCE_PROVIDER_CACHE_DISTRIBUTION_API',
     ];
 }

classes/Services/ComposerService.php

@@ -0,0 +1,66 @@
+<?php
+
+/**
+ * Copyright since 2007 PrestaShop SA and Contributors
+ * PrestaShop is an International Registered Trademark & Property of PrestaShop SA
+ *
+ * NOTICE OF LICENSE
+ *
+ * This source file is subject to the Academic Free License 3.0 (AFL-3.0)
+ * that is bundled with this package in the file LICENSE.md.
+ * It is also available through the world-wide-web at this URL:
+ * https://opensource.org/licenses/AFL-3.0
+ * If you did not receive a copy of the license and are unable to
+ * obtain it through the world-wide-web, please send an email
+ * to [email protected] so we can send you a copy immediately.
+ *
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade PrestaShop to newer
+ * versions in the future. If you wish to customize PrestaShop for your
+ * needs please refer to https://devdocs.prestashop.com/ for more information.
+ *
+ * @author    PrestaShop SA and Contributors <[email protected]>
+ * @copyright Since 2007 PrestaShop SA and Contributors
+ * @license   https://opensource.org/licenses/AFL-3.0 Academic Free License 3.0 (AFL-3.0)
+ */
+
+namespace PrestaShop\Module\AutoUpgrade\Services;
+
+class ComposerService
+{
+    const COMPOSER_PACKAGE_TYPE = 'prestashop-module';
+
+    /**
+     * Returns packages defined as PrestaShop modules in composer.lock
+     *
+     * @return array<array{name:string, version:string}>
+     */
+    public function getModulesInComposerLock(string $composerFile): array
+    {
+        if (!file_exists($composerFile)) {
+            return [];
+        }
+        // Native modules are the one integrated in PrestaShop release via composer
+        // so we use the lock files to generate the list
+        $content = file_get_contents($composerFile);
+        $content = json_decode($content, true);
+        if (empty($content['packages'])) {
+            return [];
+        }
+
+        $modules = array_filter($content['packages'], function (array $package) {
+            return self::COMPOSER_PACKAGE_TYPE === $package['type'] && !empty($package['name']);
+        });
+        $modules = array_map(function (array $package) {
+            $vendorName = explode('/', $package['name']);
+
+            return [
+                'name' => $vendorName[1],
+                'version' => ltrim($package['version'], 'v'),
+            ];
+        }, $modules);
+
+        return $modules;
+    }
+}

classes/State.php

@@ -148,27 +148,8 @@ public function export(): array
         return get_object_vars($this);
     }
 
-    public function initDefault(Upgrader $upgrader, string $prodRootDir, string $version): void
-    {
-        $postData = http_build_query([
-            'action' => 'native',
-            'iso_code' => 'all',
-            'method' => 'listing',
-            'version' => $this->getInstallVersion(),
-        ]);
-        $xml_local = $prodRootDir . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'xml' . DIRECTORY_SEPARATOR . 'modules_native_addons.xml';
-        $xml = $upgrader->getApiAddons($xml_local, $postData, true);
-
-        $modules_addons = $modules_versions = [];
-        if (is_object($xml)) {
-            foreach ($xml as $mod) {
-                $modules_addons[(string) $mod->id] = (string) $mod->name;
-                $modules_versions[(string) $mod->id] = (string) $mod->version;
-            }
-        }
-        $this->setModulesAddons($modules_addons);
-        $this->setModulesVersions($modules_versions);
-
+    public function initDefault(string $version): void
+    {
         // installedLanguagesIso is used to merge translations files
         $installedLanguagesIso = array_map(
             function ($v) { return $v['iso_code']; },
@@ -257,20 +238,6 @@ public function getInstalledLanguagesIso(): array
         return $this->installedLanguagesIso;
     }
 
-    /** @return array<string, string> Key is the module ID on Addons */
-    public function getModules_addons(): array
-    {
-        return $this->modules_addons;
-    }
-
-    /**
-     * @return array<string, string>
-     */
-    public function getModulesVersions(): array
-    {
-        return $this->modules_versions;
-    }
-
     public function getWarningExists(): bool
     {
         return $this->warning_exists;
@@ -399,16 +366,6 @@ public function setInstalledLanguagesIso(array $installedLanguagesIso): State
         return $this;
     }
 
-    /**
-     * @param array<string, string> $modules_addons Key is the module ID on Addons
-     */
-    public function setModulesAddons(array $modules_addons): State
-    {
-        $this->modules_addons = $modules_addons;
-
-        return $this;
-    }
-
     /**
      * @param array<string, string> $modules_versions
      *

classes/Task/Runner/AllUpgradeTasks.php

@@ -114,10 +114,7 @@ public function init(): void
     {
         if ($this->step === self::initialTask) {
             parent::init();
-            $this->container->getState()->initDefault(
-                $this->container->getUpgrader(),
-                $this->container->getProperty(UpgradeContainer::PS_ROOT_PATH),
-                $this->container->getProperty(UpgradeContainer::PS_VERSION));
+            $this->container->getState()->initDefault($this->container->getProperty(UpgradeContainer::PS_VERSION));
         }
     }
 }

classes/Task/Upgrade/UpgradeModules.php

@@ -41,6 +41,7 @@
 use PrestaShop\Module\AutoUpgrade\UpgradeTools\Module\ModuleUnzipper;
 use PrestaShop\Module\AutoUpgrade\UpgradeTools\Module\ModuleUnzipperContext;
 use PrestaShop\Module\AutoUpgrade\UpgradeTools\Module\ModuleVersionAdapter;
+use PrestaShop\Module\AutoUpgrade\UpgradeTools\Module\Source\ModuleSourceAggregate;
 use Symfony\Component\Filesystem\Filesystem;
 
 /**
@@ -61,61 +62,56 @@ public function run(): int
 
         $listModules = Backlog::fromContents($this->container->getFileConfigurationStorage()->load(UpgradeFileNames::MODULES_TO_UPGRADE_LIST));
 
-        // add local modules that we want to upgrade to the list
-        $localModules = $this->getLocalModules();
-        if (!empty($localModules)) {
-            foreach ($localModules as $currentLocalModule) {
-                $listModules[$currentLocalModule['name']] = [
-                    'id' => $currentLocalModule['id_module'],
-                    'name' => $currentLocalModule['name'],
-                    'is_local' => true,
-                ];
-            }
-        }
-
         $modulesPath = $this->container->getProperty(UpgradeContainer::PS_ROOT_PATH) . DIRECTORY_SEPARATOR . 'modules' . DIRECTORY_SEPARATOR;
 
-        $moduleDownloader = new ModuleDownloader($this->translator, $this->logger, $this->container->getState()->getInstallVersion());
+        $moduleSourceList = new ModuleSourceAggregate($this->container->getModuleSourceProviders());
+        $moduleDownloader = new ModuleDownloader($this->translator, $this->logger, $this->container->getProperty(UpgradeContainer::TMP_PATH));
         $moduleUnzipper = new ModuleUnzipper($this->translator, $this->container->getZipAction(), $modulesPath);
         $moduleMigration = new ModuleMigration($this->translator, $this->logger);
 
         if ($listModules->getRemainingTotal()) {
             $moduleInfos = $listModules->getNext();
 
-            $zipFullPath = $this->container->getProperty(UpgradeContainer::TMP_PATH) . DIRECTORY_SEPARATOR . $moduleInfos['name'] . '.zip';
-
             try {
-                $this->logger->debug($this->translator->trans('Updating module %module%...', ['%module%' => $moduleInfos['name']]));
+                $this->logger->debug($this->translator->trans('Checking updates of module %module%...', ['%module%' => $moduleInfos['name']]));
 
-                $moduleDownloaderContext = new ModuleDownloaderContext($zipFullPath, $moduleInfos);
-                $moduleDownloader->downloadModule($moduleDownloaderContext);
+                $moduleDownloaderContext = new ModuleDownloaderContext($moduleInfos);
+                $moduleSourceList->setSourcesIn($moduleDownloaderContext);
 
-                $moduleUnzipperContext = new ModuleUnzipperContext($zipFullPath, $moduleInfos['name']);
-                $moduleUnzipper->unzipModule($moduleUnzipperContext);
+                if (empty($moduleDownloaderContext->getUpdateSources())) {
+                    $this->logger->debug($this->translator->trans('Module %module% is up-to-date.', ['%module%' => $moduleInfos['name']]));
+                } else {
+                    $moduleDownloader->downloadModule($moduleDownloaderContext);
 
-                $dbVersion = (new ModuleVersionAdapter())->get($moduleInfos['name']);
-                $module = \Module::getInstanceByName($moduleInfos['name']);
+                    $moduleUnzipperContext = new ModuleUnzipperContext($moduleDownloaderContext->getPathToModuleUpdate(), $moduleInfos['name']);
+                    $moduleUnzipper->unzipModule($moduleUnzipperContext);
 
-                if (!($module instanceof \Module)) {
-                    throw (new UpgradeException($this->translator->trans('[WARNING] Error when trying to retrieve module %s instance.', [$moduleInfos['name']])))->setSeverity(UpgradeException::SEVERITY_WARNING);
-                }
+                    $dbVersion = (new ModuleVersionAdapter())->get($moduleInfos['name']);
+                    $module = \Module::getInstanceByName($moduleInfos['name']);
 
-                $moduleMigrationContext = new ModuleMigrationContext($module, $dbVersion);
+                    if (!($module instanceof \Module)) {
+                        throw (new UpgradeException($this->translator->trans('[WARNING] Error when trying to retrieve module %s instance.', [$moduleInfos['name']])))->setSeverity(UpgradeException::SEVERITY_WARNING);
+                    }
 
-                if (!$moduleMigration->needMigration($moduleMigrationContext)) {
-                    $this->logger->info($this->translator->trans('Module %s does not need to be migrated. Module is up to date.', [$moduleInfos['name']]));
-                } else {
-                    $moduleMigration->runMigration($moduleMigrationContext);
+                    $moduleMigrationContext = new ModuleMigrationContext($module, $dbVersion);
+
+                    if (!$moduleMigration->needMigration($moduleMigrationContext)) {
+                        $this->logger->info($this->translator->trans('Module %s does not need to be migrated. Module is up to date.', [$moduleInfos['name']]));
+                    } else {
+                        $moduleMigration->runMigration($moduleMigrationContext);
+                    }
+                    $moduleMigration->saveVersionInDb($moduleMigrationContext);
                 }
-                $moduleMigration->saveVersionInDb($moduleMigrationContext);
             } catch (UpgradeException $e) {
                 $this->handleException($e);
                 if ($e->getSeverity() === UpgradeException::SEVERITY_ERROR) {
                     return ExitCode::FAIL;
                 }
             } finally {
                 // Cleanup of module assets
-                (new Filesystem())->remove([$zipFullPath]);
+                if (!empty($moduleDownloaderContext) && !empty($moduleDownloaderContext->getPathToModuleUpdate())) {
+                    (new Filesystem())->remove([$moduleDownloaderContext->getPathToModuleUpdate()]);
+                }
             }
         }
 
@@ -128,64 +124,25 @@ public function run(): int
         if ($modules_left) {
             $this->stepDone = false;
             $this->next = 'upgradeModules';
-            $this->logger->info($this->translator->trans('%s modules left to upgrade.', [$modules_left]));
+            $this->logger->info($this->translator->trans('%s modules left to update.', [$modules_left]));
         } else {
             $this->stepDone = true;
             $this->status = 'ok';
             $this->next = 'cleanDatabase';
-            $this->logger->info($this->translator->trans('Addons modules files have been upgraded.'));
+            $this->logger->info($this->translator->trans('All modules have been updated.'));
         }
 
         return ExitCode::SUCCESS;
     }
 
-    /**
-     * Get the list of module zips in admin/autoupgrade/modules
-     * These zips will be used to upgrade related modules instead of using distant zips on addons
-     *
-     * @return array<string, mixed>
-     */
-    private function getLocalModules(): array
-    {
-        $localModuleDir = sprintf(
-            '%s%sautoupgrade%smodules',
-            _PS_ADMIN_DIR_,
-            DIRECTORY_SEPARATOR,
-            DIRECTORY_SEPARATOR
-        );
-
-        $zipFileNames = [];
-
-        $zipFiles = glob($localModuleDir . DIRECTORY_SEPARATOR . '*.zip');
-
-        if (empty($zipFiles)) {
-            return [];
-        }
-
-        foreach ($zipFiles as $zipFile) {
-            $zipFileNames[] = pSQL(pathinfo($zipFile, PATHINFO_FILENAME));
-        }
-
-        $sql = sprintf(
-            "SELECT id_module, name FROM %smodule WHERE name IN ('%s')",
-            _DB_PREFIX_,
-            implode("','", $zipFileNames)
-        );
-
-        return \Db::getInstance()->executeS($sql);
-    }
-
     public function warmUp(): int
     {
         $this->container->getState()->setProgressPercentage(
             $this->container->getCompletionCalculator()->getBasePercentageOfTask(self::class)
         );
 
         try {
-            $modulesToUpgrade = $this->container->getModuleAdapter()->listModulesToUpgrade(
-                $this->container->getState()->getModules_addons(),
-                $this->container->getState()->getModulesVersions()
-            );
+            $modulesToUpgrade = $this->container->getModuleAdapter()->listModulesPresentInFolderAndInstalled();
             $modulesToUpgrade = array_reverse($modulesToUpgrade);
             $total_modules_to_upgrade = count($modulesToUpgrade);