From fcd3c0438ff965393d7a1216a58c98824e0ee113 Mon Sep 17 00:00:00 2001 From: Jordi Boggiano Date: Tue, 3 Sep 2024 14:58:04 +0200 Subject: [PATCH] Speed up fixture loading drastically to improve the setup experience, fixes #1457 --- README.md | 17 ++- src/Command/MigrateDownloadCountsCommand.php | 2 +- src/Command/MigratePhpStatsCommand.php | 2 +- src/DataFixtures/DownloadFixtures.php | 75 +++++++---- src/DataFixtures/PackageFixtures.php | 131 ++++++------------- src/DataFixtures/UserFixtures.php | 21 ++- src/Service/UpdaterWorker.php | 13 ++ src/Util/LoggingHttpDownloader.php | 23 +++- 8 files changed, 157 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 9cab004c8..a31259758 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ These steps are provided for development purposes only. composer install npm install ``` - Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in .env.local and possibly `MAILER_DSN` if you need to receive email. + Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in `.env.local`. Set also `MAILER_DSN` if you'd like to receive email. 3. Start the web server: ```bash @@ -70,12 +70,19 @@ You should now be able to access the site, create a user, etc. You can get test data by running the fixtures: ```bash -bin/console doctrine:fixtures:load +bin/console doctrine:fixtures:load --group base +bin/console doctrine:fixtures:load --group downloads --append ``` -This will create 100 packages from packagist.org, update them from GitHub, -populate them with fake download stats, and assign a user named `dev` -(with password: `dev`) as their maintainer. +This will create some packages, update them from GitHub, populate them +with fake download stats, and assign a user named `dev` (with password: `dev`) +as their maintainer. + +There is also a user `user` (with password: `user`) that has no access if you +need to check readonly views. + +Finally there is a user `admin` (with password: `admin`) that has super admin +permissions. ### Search diff --git a/src/Command/MigrateDownloadCountsCommand.php b/src/Command/MigrateDownloadCountsCommand.php index 0cfe5472f..6aa03bbbd 100644 --- a/src/Command/MigrateDownloadCountsCommand.php +++ b/src/Command/MigrateDownloadCountsCommand.php @@ -61,7 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int try { // might be a large-ish dataset coming through here - ini_set('memory_limit', '1G'); + ini_set('memory_limit', -1); $now = new \DateTimeImmutable(); $todaySuffix = ':'.$now->format('Ymd'); diff --git a/src/Command/MigratePhpStatsCommand.php b/src/Command/MigratePhpStatsCommand.php index 06a003787..627499ae9 100644 --- a/src/Command/MigratePhpStatsCommand.php +++ b/src/Command/MigratePhpStatsCommand.php @@ -64,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int try { // might be a large-ish dataset coming through here - ini_set('memory_limit', '2G'); + ini_set('memory_limit', -1); $now = new \DateTimeImmutable(); $yesterday = new \DateTimeImmutable('yesterday'); diff --git a/src/DataFixtures/DownloadFixtures.php b/src/DataFixtures/DownloadFixtures.php index bb2cdaba8..399254302 100644 --- a/src/DataFixtures/DownloadFixtures.php +++ b/src/DataFixtures/DownloadFixtures.php @@ -4,12 +4,15 @@ namespace App\DataFixtures; +use App\Command\CompileStatsCommand; use App\Command\MigrateDownloadCountsCommand; +use App\Command\MigratePhpStatsCommand; use App\Entity\Package; use App\Entity\Version; use DateInterval; use DateTimeImmutable; use Doctrine\Bundle\FixturesBundle\Fixture; +use Doctrine\Bundle\FixturesBundle\FixtureGroupInterface; use Doctrine\Common\DataFixtures\DependentFixtureInterface; use Doctrine\ORM\EntityManagerInterface; use Doctrine\Persistence\ObjectManager; @@ -21,19 +24,19 @@ /** * Creates fake download statistics for each package. */ -class DownloadFixtures extends Fixture implements DependentFixtureInterface +class DownloadFixtures extends Fixture implements FixtureGroupInterface { public function __construct( private Client $redis, - private MigrateDownloadCountsCommand $migrateDownloadCountsCommand + private MigrateDownloadCountsCommand $migrateDownloadCountsCommand, + private readonly MigratePhpStatsCommand $migratePhpStatsCommand, + private readonly CompileStatsCommand $compileStatsCommand, ) { } - public function getDependencies(): array + public static function getGroups(): array { - return [ - PackageFixtures::class, - ]; + return ['downloads']; } public function load(ObjectManager $manager): void @@ -43,25 +46,41 @@ public function load(ObjectManager $manager): void $output->writeln('Generating downloads...'); - /** @var Package[] $packages */ - $packages = $manager->getRepository(Package::class)->findAll(); + $pkgNames = ['composer/pcre', 'monolog/monolog', 'twig/twig']; + $packages = $manager->getRepository(Package::class)->findBy(['name' => $pkgNames]); + + $versions = []; + foreach ($packages as $index => $package) { + if ($package->getName() === 'composer/pcre') { + $versions[$index] = $package->getVersions(); + } else { + assert($manager instanceof EntityManagerInterface); + $latestVersion = $this->getLatestPackageVersion($manager, $package); + $versions[$index][] = $latestVersion; + } + } + + if ($versions === []) { + echo 'No packages found, make sure to run "bin/console doctrine:fixtures:load --group base" before the download fixtures' . PHP_EOL; + return; + } - $progressBar = new ProgressBar($output, count($packages)); + echo 'Creating download fixtures for packages: '.implode(', ', $pkgNames).PHP_EOL; + + $progressBar = new ProgressBar($output, array_sum(array_map('count', $versions))); $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% (%remaining% left) %message%'); $progressBar->setMessage(''); $progressBar->start(); // Set the Redis keys that would normally be set by the DownloadManager, for the whole period. - - foreach ($packages as $package) { + foreach ($packages as $index => $package) { $progressBar->setMessage($package->getName()); $progressBar->display(); - /** @var EntityManagerInterface $manager */ - $latestVersion = $this->getLatestPackageVersion($manager, $package); - - $this->populateDownloads($package, $latestVersion); + foreach ($versions[$index] as $version) { + $this->populateDownloads($package, $version); + } $progressBar->advance(); } @@ -69,10 +88,13 @@ public function load(ObjectManager $manager): void $progressBar->finish(); $output->writeln(''); - // Then migrate the Redis keys to the db + $manager->clear(); + // Then migrate the Redis keys to the db $output->writeln('Migrating downloads to db... (this may take some time)'); $this->migrateDownloadCountsCommand->run($input, $output); + $this->migratePhpStatsCommand->run($input, $output); + $this->compileStatsCommand->run($input, $output); } /** @@ -115,19 +137,22 @@ private function populateDownloads(Package $package, Version $version): void $day = $date->format('Ymd'); $month = $date->format('Ym'); + $phpMinorPlatform = random_int(7, 8).'.'.random_int(0, 4); + $keys = [ - 'downloads', - 'downloads:' . $day, - 'downloads:' . $month, + 'downloads' => $downloads, + 'downloads:' . $day => $downloads, + 'downloads:' . $month => $downloads, + + 'dl:' . $package->getId() => $downloads, + 'dl:' . $package->getId() . ':' . $day => $downloads, + 'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day => $downloads, - 'dl:' . $package->getId(), - 'dl:' . $package->getId() . ':' . $day, - 'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day + 'phpplatform:'.$phpMinorPlatform.':' => $downloads, + 'phpplatform:'.$package->getId() . '-' . $version->getId().':'.$phpMinorPlatform.':'.$day => $downloads, ]; - foreach ($keys as $key) { - $this->redis->incrby($key, $downloads); - } + $this->redis->mset($keys); $date = $date->add(new DateInterval('P1D')); diff --git a/src/DataFixtures/PackageFixtures.php b/src/DataFixtures/PackageFixtures.php index 3f1c6409f..bcea26aca 100644 --- a/src/DataFixtures/PackageFixtures.php +++ b/src/DataFixtures/PackageFixtures.php @@ -12,6 +12,7 @@ use App\Service\UpdaterWorker; use DateTime; use Doctrine\Bundle\FixturesBundle\Fixture; +use Doctrine\Bundle\FixturesBundle\FixtureGroupInterface; use Doctrine\Common\DataFixtures\DependentFixtureInterface; use Doctrine\Persistence\ObjectManager; use Monolog\Logger; @@ -22,7 +23,7 @@ /** * Creates packages and updates them from GitHub. */ -class PackageFixtures extends Fixture implements DependentFixtureInterface +class PackageFixtures extends Fixture implements DependentFixtureInterface, FixtureGroupInterface { private SignalHandler $signalHandler; @@ -34,6 +35,11 @@ public function __construct( $this->signalHandler = SignalHandler::create(null, $logger); } + public static function getGroups(): array + { + return ['base']; + } + public function getDependencies(): array { return [ @@ -47,6 +53,8 @@ public function load(ObjectManager $manager): void $packages = $this->getPackages(); + echo 'Creating '.count($packages).' packages. "composer/pcre" has full version information, the others only one branch and one tag each.' . PHP_EOL; + $progressBar = new ProgressBar($output, count($packages)); $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% (%remaining% left) %message%'); @@ -57,6 +65,7 @@ public function load(ObjectManager $manager): void $maintainer = $this->getReference(UserFixtures::PACKAGE_MAINTAINER); foreach ($packages as [$repoUrl, $createdAt]) { + $mtime = microtime(true); /** * The EntityManager gets cleared by the UpdaterWorker, so the User becomes detached. * We need to re-load into the current EntityManager on every loop iteration. @@ -73,18 +82,25 @@ public function load(ObjectManager $manager): void $package->addMaintainer($maintainer); $package->setRepository($repoUrl); + if (!$package->getName()) { + var_dump($repoUrl.' needs to be updated or removed in '.__FILE__.' as it is not loadable anymore'); + continue; + } + $manager->getRepository(Vendor::class)->createIfNotExists($package->getVendor()); $manager->persist($package); $this->providerManager->insertPackage($package); $manager->flush(); + $this->updaterWorker->setLoadMinimalVersions($package->getName() !== 'composer/pcre'); $this->updatePackage($package->getId()); $progressBar->advance(); } $progressBar->finish(); + $manager->clear(); $output->writeln(''); } @@ -107,106 +123,37 @@ private function updatePackage(int $id): void private function getPackages(): array { return [ - ['https://github.com/php-fig/log', '2012-12-21T08:45:50+00:00'], - ['https://github.com/symfony/polyfill-mbstring', '2015-10-25T13:17:47+00:00'], - ['https://github.com/symfony/console', '2011-10-16T03:41:36+00:00'], - ['https://github.com/symfony/event-dispatcher', '2011-10-16T03:42:08+00:00'], + ['https://github.com/composer/pcre', '2016-04-11T15:12:41+00:00'], + ['https://github.com/Seldaek/monolog', '2011-09-27T00:35:19+00:00'], + ['https://github.com/briannesbitt/Carbon', '2012-09-11T02:06:50+00:00'], + ['https://github.com/doctrine/event-manager', '2018-06-07T14:18:48+00:00'], + ['https://github.com/doctrine/inflector', '2013-01-10T21:54:25+00:00'], ['https://github.com/doctrine/instantiator', '2014-08-12T01:08:01+00:00'], - ['https://github.com/symfony/finder', '2011-10-16T03:42:15+00:00'], - ['https://github.com/guzzle/guzzle', '2011-11-09T04:33:13+00:00'], + ['https://github.com/firebase/php-jwt', '2013-08-30T21:20:41+00:00'], + ['https://github.com/jmespath/jmespath.php', '2013-11-27T00:36:44+00:00'], ['https://github.com/php-fig/http-message', '2014-06-10T23:09:12+00:00'], - ['https://github.com/symfony/process', '2011-10-16T03:42:53+00:00'], - ['https://github.com/guzzle/psr7', '2015-03-05T23:21:09+00:00'], - ['https://github.com/doctrine/inflector', '2013-01-10T21:54:25+00:00'], - ['https://github.com/sebastianbergmann/phpunit', '2012-09-18T06:46:25+00:00'], - ['https://github.com/phpDocumentor/ReflectionDocBlock', '2012-05-08T17:56:00+00:00'], - ['https://github.com/Seldaek/monolog', '2011-09-27T00:35:19+00:00'], + ['https://github.com/php-fig/log', '2012-12-21T08:45:50+00:00'], + ['https://github.com/phpseclib/phpseclib', '2012-06-10T05:33:10+00:00'], + ['https://github.com/phpspec/prophecy', '2013-03-25T11:44:35+00:00'], + ['https://github.com/ramsey/uuid', '2015-04-25T19:44:46+00:00'], + ['https://github.com/schmittjoh/php-option', '2012-11-05T16:24:14+00:00,'], + ['https://github.com/sebastianbergmann/comparator', '2014-01-22T07:47:33+00:00'], + ['https://github.com/sebastianbergmann/diff', '2013-02-12T10:19:12+00:00'], + ['https://github.com/sebastianbergmann/environment', '2014-02-10T15:56:05+00:00'], + ['https://github.com/sebastianbergmann/exporter', '2013-02-16T10:01:17+00:00'], + ['https://github.com/sebastianbergmann/global-state', '2014-08-23T08:01:04+00:00'], ['https://github.com/sebastianbergmann/php-code-coverage', '2012-09-18T06:59:55+00:00'], - ['https://github.com/doctrine/lexer', '2013-01-12T19:00:31+00:00'], - ['https://github.com/sebastianbergmann/php-timer', '2012-09-18T06:58:26+00:00'], ['https://github.com/sebastianbergmann/php-file-iterator', '2012-09-18T06:59:23+00:00'], - ['https://github.com/sebastianbergmann/diff', '2013-02-12T10:19:12+00:00'], - ['https://github.com/guzzle/promises', '2015-02-25T20:24:35+00:00'], ['https://github.com/sebastianbergmann/php-text-template', '2012-08-01T15:59:49+00:00'], - ['https://github.com/sebastianbergmann/exporter', '2013-02-16T10:01:17+00:00'], - ['https://github.com/sebastianbergmann/environment', '2014-02-10T15:56:05+00:00'], + ['https://github.com/sebastianbergmann/php-timer', '2012-09-18T06:58:26+00:00'], ['https://github.com/sebastianbergmann/php-token-stream', '2012-09-18T06:57:55+00:00'], - ['https://github.com/phpspec/prophecy', '2013-03-25T11:44:35+00:00'], - ['https://github.com/paragonie/random_compat', '2015-07-07T20:20:09+00:00'], - ['https://github.com/sebastianbergmann/version', '2013-01-05T14:28:55+00:00'], - ['https://github.com/sebastianbergmann/recursion-context', '2015-01-23T07:01:59+00:00'], - ['https://github.com/webmozarts/assert', '2015-03-11T12:18:50+00:00'], - ['https://github.com/symfony/debug', '2013-04-07T19:24:19+00:00'], - ['https://github.com/symfony/http-foundation', '2011-10-16T03:42:31+00:00'], - ['https://github.com/sebastianbergmann/global-state', '2014-08-23T08:01:04+00:00'], - ['https://github.com/symfony/translation', '2011-10-16T03:43:49+00:00'], - ['https://github.com/symfony/yaml', '2011-10-16T03:44:01+00:00'], - ['https://github.com/phpDocumentor/TypeResolver', '2015-06-12T17:40:32+00:00'], - ['https://github.com/swiftmailer/swiftmailer', '2011-09-29T17:10:04+00:00'], - ['https://github.com/phpDocumentor/ReflectionCommon', '2015-06-05T08:49:50+00:00'], - ['https://github.com/php-fig/container', '2016-09-08T23:35:56+00:00'], - ['https://github.com/symfony/polyfill-ctype', '2018-05-01T05:39:10+00:00'], - ['https://github.com/symfony/http-kernel', '2011-10-16T03:42:38+00:00'], - ['https://github.com/nikic/PHP-Parser', '2012-01-05T12:46:24+00:00'], - ['https://github.com/sebastianbergmann/comparator', '2014-01-22T07:47:33+00:00'], - ['https://github.com/symfony/css-selector', '2011-10-16T03:41:43+00:00'], - ['https://github.com/symfony/var-dumper', '2014-09-26T12:59:44+00:00'], - ['https://github.com/myclabs/DeepCopy', '2013-07-22T15:50:26+00:00'], - ['https://github.com/doctrine/cache', '2013-01-10T22:44:33+00:00'], - ['https://github.com/doctrine/annotations', '2013-01-12T19:24:37+00:00'], - ['https://github.com/sebastianbergmann/resource-operations', '2015-07-19T06:47:16+00:00'], - ['https://github.com/sebastianbergmann/code-unit-reverse-lookup', '2016-02-08T20:26:02+00:00'], - ['https://github.com/sebastianbergmann/object-enumerator', '2016-01-28T06:41:48+00:00'], - ['https://github.com/symfony/filesystem', '2012-01-10T14:22:45+00:00'], - ['https://github.com/briannesbitt/Carbon', '2012-09-11T02:06:50+00:00'], - ['https://github.com/symfony/routing', '2011-10-16T03:43:03+00:00'], - ['https://github.com/doctrine/dbal', '2011-10-10T17:32:36+00:00'], ['https://github.com/sebastianbergmann/phpunit-mock-objects', '2012-09-18T06:55:43+00:00'], - ['https://github.com/ramsey/uuid', '2015-04-25T19:44:46+00:00'], - ['https://github.com/symfony/polyfill-php72', '2017-06-09T14:29:13+00:00'], + ['https://github.com/sebastianbergmann/recursion-context', '2015-01-23T07:01:59+00:00'], + ['https://github.com/sebastianbergmann/version', '2013-01-05T14:28:55+00:00'], + ['https://github.com/squizlabs/PHP_CodeSniffer', '2012-11-06T03:18:51+00:00'], ['https://github.com/thephpleague/flysystem', '2014-01-15T07:46:47+00:00'], - ['https://github.com/php-fig/simple-cache', '2016-11-04T11:18:34+00:00'], - ['https://github.com/vlucas/phpdotenv', '2013-01-23T07:02:02+00:00'], - ['https://github.com/doctrine/collections', '2013-01-12T16:39:32+00:00'], - ['https://github.com/ralouphie/getallheaders', '2014-06-02T21:22:26+00:00'], - ['https://github.com/doctrine/common', '2011-10-10T17:32:26+00:00'], ['https://github.com/twigphp/Twig', '2011-09-29T16:52:42+00:00'], - ['https://github.com/fzaninotto/Faker', '2011-11-09T14:18:39+00:00'], - ['https://github.com/theseer/tokenizer', '2017-04-05T18:34:08+00:00'], - ['https://github.com/sebastianbergmann/object-reflector', '2017-03-12T15:13:02+00:00'], - ['https://github.com/phar-io/version', '2016-11-30T11:36:58+00:00'], - ['https://github.com/phar-io/manifest', '2016-11-24T16:53:22+00:00'], - ['https://github.com/bobthecow/psysh', '2013-06-21T15:04:58+00:00'], - ['https://github.com/laravel/framework', '2013-01-10T21:31:35+00:00'], - ['https://github.com/dnoegel/php-xdg-base-dir', '2014-06-30T18:37:36+00:00'], - ['https://github.com/aws/aws-sdk-php', '2012-11-02T18:05:04+00:00'], - ['https://github.com/php-fig/cache', '2015-12-11T11:25:13+00:00'], - ['https://github.com/symfony/polyfill-php70', '2015-10-25T13:15:41+00:00'], - ['https://github.com/mockery/mockery', '2012-01-21T22:10:42+00:00'], - ['https://github.com/symfony/dom-crawler', '2011-10-16T03:42:00+00:00'], - ['https://github.com/symfony/polyfill-intl-idn', '2018-10-01T13:07:00+00:00'], - ['https://github.com/symfony/config', '2011-10-16T03:41:26+00:00'], - ['https://github.com/hamcrest/hamcrest-php', '2013-12-31T06:34:20+00:00'], - ['https://github.com/JakubOnderka/PHP-Console-Color', '2014-04-07T19:56:56+00:00'], - ['https://github.com/JakubOnderka/PHP-Console-Highlighter', '2013-11-23T13:24:12+00:00'], - ['https://github.com/egulias/EmailValidator', '2013-05-19T17:04:12+00:00'], - ['https://github.com/symfony/polyfill-php56', '2015-10-25T13:16:06+00:00'], - ['https://github.com/symfony/polyfill-php73', '2018-04-25T16:25:32+00:00'], - ['https://github.com/symfony/service-contracts', '2019-05-27T07:16:14+00:00'], - ['https://github.com/tijsverkoyen/CssToInlineStyles', '2012-09-30T14:34:16+00:00'], - ['https://github.com/symfony/polyfill-util', '2015-10-25T13:17:23+00:00'], - ['https://github.com/schmittjoh/php-option', '2012-11-05T16:24:14+00:00,'], - ['https://github.com/phpseclib/phpseclib', '2012-06-10T05:33:10+00:00'], - ['https://github.com/symfony/dependency-injection', '2011-10-16T03:41:53+00:00'], - ['https://github.com/jmespath/jmespath.php', '2013-11-27T00:36:44+00:00'], - ['https://github.com/doctrine/orm', '2011-10-10T17:32:08+00:00'], - ['https://github.com/composer/ca-bundle', '2016-04-11T15:12:41+00:00'], - ['https://github.com/squizlabs/PHP_CodeSniffer', '2012-11-06T03:18:51+00:00'], - ['https://github.com/symfony/event-dispatcher-contracts', '2019-05-27T07:08:51+00:00'], - ['https://github.com/firebase/php-jwt', '2013-08-30T21:20:41+00:00'], - ['https://github.com/doctrine/event-manager', '2018-06-07T14:18:48+00:00'], - ['https://github.com/symfony/options-resolver', '2012-05-16T06:01:18+00:00'], - ['https://github.com/symfony/polyfill-iconv', '2015-10-25T13:15:09+00:00'], + ['https://github.com/webmozarts/assert', '2015-03-11T12:18:50+00:00'], ]; } } diff --git a/src/DataFixtures/UserFixtures.php b/src/DataFixtures/UserFixtures.php index 1fbd559c8..9e4e179eb 100644 --- a/src/DataFixtures/UserFixtures.php +++ b/src/DataFixtures/UserFixtures.php @@ -6,13 +6,14 @@ use App\Entity\User; use Doctrine\Bundle\FixturesBundle\Fixture; +use Doctrine\Bundle\FixturesBundle\FixtureGroupInterface; use Doctrine\Persistence\ObjectManager; use Symfony\Component\PasswordHasher\Hasher\UserPasswordHasherInterface; /** * Creates a user to be assigned as the maintainer of packages. */ -class UserFixtures extends Fixture +class UserFixtures extends Fixture implements FixtureGroupInterface { public const PACKAGE_MAINTAINER = 'package-maintainer'; @@ -20,14 +21,30 @@ public function __construct(private UserPasswordHasherInterface $passwordHasher) { } + public static function getGroups(): array + { + return ['base']; + } + public function load(ObjectManager $manager): void { + echo 'Creating users admin (password: admin), dev (password: dev), and user (password: user)'.PHP_EOL; + + $dev = new User; + $dev->setEmail('admin@example.org'); + $dev->setUsername('admin'); + $dev->setPassword($this->passwordHasher->hashPassword($dev, 'admin')); + $dev->setEnabled(true); + $dev->setRoles(['ROLE_SUPERADMIN']); + + $manager->persist($dev); + $dev = new User; $dev->setEmail('dev@example.org'); $dev->setUsername('dev'); $dev->setPassword($this->passwordHasher->hashPassword($dev, 'dev')); $dev->setEnabled(true); - $dev->setRoles(['ROLE_SUPERADMIN']); + $dev->setRoles([]); $manager->persist($dev); diff --git a/src/Service/UpdaterWorker.php b/src/Service/UpdaterWorker.php index 1d815dda5..605df7d84 100644 --- a/src/Service/UpdaterWorker.php +++ b/src/Service/UpdaterWorker.php @@ -62,6 +62,8 @@ class UpdaterWorker private Scheduler $scheduler; private PackageManager $packageManager; private DownloadManager $downloadManager; + /** For use in fixtures loader only */ + private bool $loadMinimalVersions = false; public function __construct( LoggerInterface $logger, @@ -83,6 +85,14 @@ public function __construct( $this->downloadManager = $downloadManager; } + /** + * @internal for fixtures usage only + */ + public function setLoadMinimalVersions(bool $loadMinimalVersions): void + { + $this->loadMinimalVersions = $loadMinimalVersions; + } + /** * @param Job $job * @return array{status: Job::STATUS_*, message?: string, after?: \DateTimeInterface, vendor?: string, details?: string, exception?: \Throwable} @@ -161,6 +171,9 @@ public function process(Job $job, SignalHandler $signal): array } $httpDownloader = new LoggingHttpDownloader($io, $config, $this->statsd, $usesPackagistToken, $packageVendor); + if ($this->loadMinimalVersions) { + $httpDownloader->loadMinimalVersions(); + } try { $flags = 0; diff --git a/src/Util/LoggingHttpDownloader.php b/src/Util/LoggingHttpDownloader.php index 2786c80b2..fb2b55c5b 100644 --- a/src/Util/LoggingHttpDownloader.php +++ b/src/Util/LoggingHttpDownloader.php @@ -14,6 +14,7 @@ use Composer\Config; use Composer\IO\IOInterface; +use Composer\Pcre\Preg; use Composer\Util\Http\Response; use Composer\Util\HttpDownloader; use Graze\DogStatsD\Client as StatsDClient; @@ -21,6 +22,9 @@ class LoggingHttpDownloader extends HttpDownloader { + /** For use in fixtures loading only */ + private bool $loadMinimalVersions = false; + public function __construct( IOInterface $io, Config $config, @@ -35,7 +39,16 @@ public function get($url, $options = []): Response { $this->track($url); - return parent::get($url, $options); + $result = parent::get($url, $options); + + if ($this->loadMinimalVersions && Preg::isMatch('{/(tags|git/refs/heads)(\?|$)}', $url)) { + $reflProp = new \ReflectionProperty(Response::class, 'request'); + $newBody = $result->decodeJson(); + $newBody = array_slice($newBody, 0, 1); + $result = new Response($reflProp->getValue($result), $result->getStatusCode(), [], (string) json_encode($newBody)); + } + + return $result; } public function add($url, $options = []): PromiseInterface @@ -59,6 +72,14 @@ public function addCopy($url, $to, $options = []): PromiseInterface return parent::addCopy($url, $to, $options); } + /** + * @internal for use in fixtures only + */ + public function loadMinimalVersions(): void + { + $this->loadMinimalVersions = true; + } + private function track(string $url): void { if (!str_starts_with($url, 'https://api.github.com/')) {