Speed up fixture loading drastically to improve the setup experience, f…

…ixes #1457

README.md

@@ -33,7 +33,7 @@ These steps are provided for development purposes only.
    composer install
    npm install
    ```
-   Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in .env.local and possibly `MAILER_DSN` if you need to receive email.
+   Ensure env vars are set up correctly, you probably need to set `APP_MAILER_FROM_EMAIL`, `APP_MAILER_FROM_NAME` and `APP_DEV_EMAIL_RECIPIENT` in `.env.local`. Set also `MAILER_DSN` if you'd like to receive email.
 
 3. Start the web server:
    ```bash
@@ -70,12 +70,19 @@ You should now be able to access the site, create a user, etc.
 You can get test data by running the fixtures:
 
 ```bash
-bin/console doctrine:fixtures:load
+bin/console doctrine:fixtures:load --group base
+bin/console doctrine:fixtures:load --group downloads --append
  ```
 
-This will create 100 packages from packagist.org, update them from GitHub,
-populate them with fake download stats, and assign a user named `dev`
-(with password: `dev`) as their maintainer.
+This will create some packages, update them from GitHub, populate them
+with fake download stats, and assign a user named `dev` (with password: `dev`)
+as their maintainer.
+
+There is also a user `user` (with password: `user`) that has no access if you
+need to check readonly views.
+
+Finally there is a user `admin` (with password: `admin`) that has super admin
+permissions.
 
 ### Search
 

src/Command/MigrateDownloadCountsCommand.php

@@ -61,7 +61,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         try {
             // might be a large-ish dataset coming through here
-            ini_set('memory_limit', '1G');
+            ini_set('memory_limit', -1);
 
             $now = new \DateTimeImmutable();
             $todaySuffix = ':'.$now->format('Ymd');

src/Command/MigratePhpStatsCommand.php

@@ -64,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         try {
             // might be a large-ish dataset coming through here
-            ini_set('memory_limit', '2G');
+            ini_set('memory_limit', -1);
 
             $now = new \DateTimeImmutable();
             $yesterday = new \DateTimeImmutable('yesterday');

src/DataFixtures/DownloadFixtures.php

@@ -4,12 +4,15 @@
 
 namespace App\DataFixtures;
 
+use App\Command\CompileStatsCommand;
 use App\Command\MigrateDownloadCountsCommand;
+use App\Command\MigratePhpStatsCommand;
 use App\Entity\Package;
 use App\Entity\Version;
 use DateInterval;
 use DateTimeImmutable;
 use Doctrine\Bundle\FixturesBundle\Fixture;
+use Doctrine\Bundle\FixturesBundle\FixtureGroupInterface;
 use Doctrine\Common\DataFixtures\DependentFixtureInterface;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\Persistence\ObjectManager;
@@ -21,19 +24,19 @@
 /**
  * Creates fake download statistics for each package.
  */
-class DownloadFixtures extends Fixture implements DependentFixtureInterface
+class DownloadFixtures extends Fixture implements FixtureGroupInterface
 {
     public function __construct(
         private Client $redis,
-        private MigrateDownloadCountsCommand $migrateDownloadCountsCommand
+        private MigrateDownloadCountsCommand $migrateDownloadCountsCommand,
+        private readonly MigratePhpStatsCommand $migratePhpStatsCommand,
+        private readonly CompileStatsCommand $compileStatsCommand,
     ) {
     }
 
-    public function getDependencies(): array
+    public static function getGroups(): array
     {
-        return [
-            PackageFixtures::class,
-        ];
+        return ['downloads'];
     }
 
     public function load(ObjectManager $manager): void
@@ -43,36 +46,55 @@ public function load(ObjectManager $manager): void
 
         $output->writeln('Generating downloads...');
 
-        /** @var Package[] $packages */
-        $packages = $manager->getRepository(Package::class)->findAll();
+        $pkgNames = ['composer/pcre', 'monolog/monolog', 'twig/twig'];
+        $packages = $manager->getRepository(Package::class)->findBy(['name' => $pkgNames]);
+
+        $versions = [];
+        foreach ($packages as $index => $package) {
+            if ($package->getName() === 'composer/pcre') {
+                $versions[$index] = $package->getVersions();
+            } else {
+                assert($manager instanceof EntityManagerInterface);
+                $latestVersion = $this->getLatestPackageVersion($manager, $package);
+                $versions[$index][] = $latestVersion;
+            }
+        }
+
+        if ($versions === []) {
+            echo 'No packages found, make sure to run "bin/console doctrine:fixtures:load --group base" before the download fixtures' . PHP_EOL;
+            return;
+        }
 
-        $progressBar = new ProgressBar($output, count($packages));
+        echo 'Creating download fixtures for packages: '.implode(', ', $pkgNames).PHP_EOL;
+
+        $progressBar = new ProgressBar($output, array_sum(array_map('count', $versions)));
         $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% (%remaining% left) %message%');
 
         $progressBar->setMessage('');
         $progressBar->start();
 
         // Set the Redis keys that would normally be set by the DownloadManager, for the whole period.
-
-        foreach ($packages as $package) {
+        foreach ($packages as $index => $package) {
             $progressBar->setMessage($package->getName());
             $progressBar->display();
 
-            /** @var EntityManagerInterface $manager */
-            $latestVersion = $this->getLatestPackageVersion($manager, $package);
-
-            $this->populateDownloads($package, $latestVersion);
+            foreach ($versions[$index] as $version) {
+                $this->populateDownloads($package, $version);
+            }
 
             $progressBar->advance();
         }
 
         $progressBar->finish();
         $output->writeln('');
 
-        // Then migrate the Redis keys to the db
+        $manager->clear();
 
+        // Then migrate the Redis keys to the db
         $output->writeln('Migrating downloads to db... (this may take some time)');
         $this->migrateDownloadCountsCommand->run($input, $output);
+        $this->migratePhpStatsCommand->run($input, $output);
+        $this->compileStatsCommand->run($input, $output);
     }
 
     /**
@@ -115,19 +137,22 @@ private function populateDownloads(Package $package, Version $version): void
             $day = $date->format('Ymd');
             $month = $date->format('Ym');
 
+            $phpMinorPlatform = random_int(7, 8).'.'.random_int(0, 4);
+
             $keys = [
-                'downloads',
-                'downloads:' . $day,
-                'downloads:' . $month,
+                'downloads' => $downloads,
+                'downloads:' . $day => $downloads,
+                'downloads:' . $month => $downloads,
+
+                'dl:' . $package->getId() => $downloads,
+                'dl:' . $package->getId() . ':' . $day => $downloads,
+                'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day => $downloads,
 
-                'dl:' . $package->getId(),
-                'dl:' . $package->getId() . ':' . $day,
-                'dl:' . $package->getId() . '-' . $version->getId() . ':' . $day
+                'phpplatform:'.$phpMinorPlatform.':' => $downloads,
+                'phpplatform:'.$package->getId() . '-' . $version->getId().':'.$phpMinorPlatform.':'.$day => $downloads,
             ];
 
-            foreach ($keys as $key) {
-                $this->redis->incrby($key, $downloads);
-            }
+            $this->redis->mset($keys);
 
             $date = $date->add(new DateInterval('P1D'));