diff --git a/src/Command/ClearVersionsCommand.php b/src/Command/ClearVersionsCommand.php index f0feffe41..97ac5c020 100644 --- a/src/Command/ClearVersionsCommand.php +++ b/src/Command/ClearVersionsCommand.php @@ -115,7 +115,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int foreach ($packageNames as $name) { $package = $packageRepo->findOneBy(['name' => $name]); if ($package !== null) { - $package->setCrawledAt(new \DateTime); + $package->setCrawledAt(new \DateTimeImmutable()); $em->persist($package); } } diff --git a/src/Command/DumpPackagesCommand.php b/src/Command/DumpPackagesCommand.php index 3bb712056..30332164c 100644 --- a/src/Command/DumpPackagesCommand.php +++ b/src/Command/DumpPackagesCommand.php @@ -99,7 +99,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int SELECT p.id FROM package p LEFT JOIN download d ON (d.id = p.id AND d.type = 1) - WHERE (replacementPackage != "spam/spam" OR replacementPackage IS NULL) + WHERE p.frozen IS NULL AND (d.total > 1000 OR d.lastUpdated > :date) ORDER BY p.id ASC ', ['date' => date('Y-m-d H:i:s', strtotime('-4months'))]); diff --git a/src/Command/DumpPackagesV2Command.php b/src/Command/DumpPackagesV2Command.php index 152210026..5973f3a12 100644 --- a/src/Command/DumpPackagesV2Command.php +++ b/src/Command/DumpPackagesV2Command.php @@ -100,7 +100,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int try { while ($iterations--) { if ($force) { - $ids = $this->getEM()->getConnection()->fetchFirstColumn('SELECT id FROM package WHERE replacementPackage != "spam/spam" OR replacementPackage IS NULL ORDER BY id ASC'); + $ids = $this->getEM()->getConnection()->fetchFirstColumn('SELECT id FROM package WHERE frozen IS NULL ORDER BY id ASC'); } else { $ids = $this->getEM()->getRepository(Package::class)->getStalePackagesForDumpingV2(); } diff --git a/src/Command/IndexPackagesCommand.php b/src/Command/IndexPackagesCommand.php index 2dc48b44e..9f1900a5a 100644 --- a/src/Command/IndexPackagesCommand.php +++ b/src/Command/IndexPackagesCommand.php @@ -14,6 +14,7 @@ use Algolia\AlgoliaSearch\SearchClient; use App\Entity\Package; +use App\Entity\PackageFreezeReason; use App\Model\DownloadManager; use App\Model\FavoriteManager; use Composer\Pcre\Preg; @@ -138,7 +139,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } // delete spam packages from the search index - if ($package->isAbandoned() && $package->getReplacementPackage() === 'spam/spam') { + if ($package->isFrozen() && $package->getFreezeReason() === PackageFreezeReason::Spam) { try { $index->deleteObject($package->getName()); $idsToUpdate[] = $package->getId(); diff --git a/src/Controller/PackageController.php b/src/Controller/PackageController.php index d7f392fd5..af40b0f3f 100644 --- a/src/Controller/PackageController.php +++ b/src/Controller/PackageController.php @@ -13,6 +13,7 @@ namespace App\Controller; use App\Entity\Dependent; +use App\Entity\PackageFreezeReason; use App\Entity\PhpStat; use App\Security\Voter\PackageActions; use App\SecurityAdvisory\GitHubSecurityAdvisoriesSource; @@ -462,7 +463,7 @@ public function markSafeAction(Request $req): RedirectResponse #[IsGranted('ROLE_ADMIN')] #[Route(path: '/package/{name}/unfreeze', name: 'unfreeze_package', requirements: ['name' => '[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+?'], defaults: ['_format' => 'html'], methods: ['POST'])] - public function unfreezePackageAction(Request $req, string $name, CsrfTokenManagerInterface $csrfTokenManager): RedirectResponse + public function unfreezePackageAction(Request $req, string $name, CsrfTokenManagerInterface $csrfTokenManager): Response { if (!$this->isCsrfTokenValid('unfreeze', (string) $req->request->get('token'))) { throw new BadRequestHttpException('Invalid CSRF token'); @@ -508,7 +509,7 @@ public function viewPackageAction(Request $req, string $name, CsrfTokenManagerIn return $package; } - if ($package->isAbandoned() && $package->getReplacementPackage() === 'spam/spam' && !$this->isGranted('ROLE_ADMIN')) { + if ($package->isFrozen() && $package->getFreezeReason() === PackageFreezeReason::Spam && !$this->isGranted('ROLE_ANTISPAM')) { throw new NotFoundHttpException('This is a spam package'); } @@ -818,7 +819,7 @@ public function updatePackageAction(Request $req, string $name): Response return new JsonResponse(['status' => 'error', 'message' => 'Package not found'], 404); } - if ($package->isAbandoned() && $package->getReplacementPackage() === 'spam/spam') { + if ($package->isFrozen() && $package->getFreezeReason() === PackageFreezeReason::Spam) { throw new NotFoundHttpException('This is a spam package'); } @@ -1017,8 +1018,8 @@ public function abandonAction(Request $request, #[MapEntity] Package $package, # $package->setAbandoned(true); $package->setReplacementPackage(str_replace('https://packagist.org/packages/', '', (string) $form->get('replacement')->getData())); $package->setIndexedAt(null); - $package->setCrawledAt(new DateTime()); - $package->setUpdatedAt(new DateTime()); + $package->setCrawledAt(new DateTimeImmutable()); + $package->setUpdatedAt(new DateTimeImmutable()); $package->setDumpedAt(null); $package->setDumpedAtV2(null); @@ -1042,8 +1043,8 @@ public function unabandonAction(#[MapEntity] Package $package, #[CurrentUser] ?U $package->setAbandoned(false); $package->setReplacementPackage(null); $package->setIndexedAt(null); - $package->setCrawledAt(new DateTime()); - $package->setUpdatedAt(new DateTime()); + $package->setCrawledAt(new DateTimeImmutable()); + $package->setUpdatedAt(new DateTimeImmutable()); $package->setDumpedAt(null); $package->setDumpedAtV2(null); diff --git a/src/Controller/UserController.php b/src/Controller/UserController.php index 5b6cf39a8..10a78eee8 100644 --- a/src/Controller/UserController.php +++ b/src/Controller/UserController.php @@ -105,7 +105,7 @@ public function markSpammerAction(Request $req, #[VarName('name')] User $user): $em->getConnection()->executeStatement( 'UPDATE package p JOIN maintainers_packages mp ON mp.package_id = p.id - SET abandoned = 1, replacementPackage = "spam/spam", suspect = "spam", indexedAt = NULL, dumpedAt = "2100-01-01 00:00:00" + SET frozen = "spam", indexedAt = NULL WHERE mp.user_id = :userId', ['userId' => $user->getId()] ); diff --git a/src/Entity/Package.php b/src/Entity/Package.php index 46886ef0c..d280cf594 100644 --- a/src/Entity/Package.php +++ b/src/Entity/Package.php @@ -37,6 +37,7 @@ enum PackageFreezeReason: string { case Spam = 'spam'; case RemoteIdMismatch = 'remote_id'; + case Gone = 'gone'; } /** @@ -52,8 +53,9 @@ enum PackageFreezeReason: string #[ORM\Index(name: 'dumped2_idx', columns: ['dumpedAtV2'])] #[ORM\Index(name: 'repository_idx', columns: ['repository'])] #[ORM\Index(name: 'remoteid_idx', columns: ['remoteId'])] -#[ORM\Index(name: 'dumped2_crawled_idx', columns: ['dumpedAtV2', 'crawledAt'])] +#[ORM\Index(name: 'dumped2_crawled_frozen_idx', columns: ['dumpedAtV2', 'crawledAt', 'frozen'])] #[ORM\Index(name: 'vendor_idx', columns: ['vendor'])] +#[ORM\Index(name: 'frozen_idx', columns: ['frozen'])] #[UniquePackage(groups: ['Create'])] #[VendorWritable(groups: ['Create'])] #[ValidPackageRepository(groups: ['Update', 'Default'])] @@ -590,7 +592,7 @@ public function getUpdatedAt(): ?DateTimeInterface public function wasUpdatedInTheLast24Hours(): bool { - return $this->updatedAt && $this->updatedAt > new \DateTime('-24 hours'); + return $this->updatedAt && $this->updatedAt > new \DateTimeImmutable('-24 hours'); } public function setCrawledAt(?DateTimeInterface $crawledAt): void @@ -735,9 +737,10 @@ public function getSuspect(): ?string public function freeze(PackageFreezeReason $reason): void { $this->frozen = $reason; - $this->setCrawledAt($dt = new \DateTimeImmutable('2100-01-01 00:00:00')); - $this->setDumpedAt($dt); - $this->setDumpedAtV2($dt); + // force re-indexing for spam packages to ensure they get deleted from the search index + if ($reason === PackageFreezeReason::Spam) { + $this->setIndexedAt(null); + } } public function unfreeze(): void @@ -747,8 +750,6 @@ public function unfreeze(): void } $this->frozen = null; $this->setCrawledAt(null); - $this->setDumpedAt(null); - $this->setDumpedAtV2(null); } public function isFrozen(): bool diff --git a/src/Entity/PackageRepository.php b/src/Entity/PackageRepository.php index d04d51da4..093f9a932 100644 --- a/src/Entity/PackageRepository.php +++ b/src/Entity/PackageRepository.php @@ -57,7 +57,7 @@ public function getPackageNamesUpdatedSince(\DateTimeInterface $date): array $query = $this->getEntityManager() ->createQuery(" SELECT p.name FROM App\Entity\Package p - WHERE p.dumpedAt >= :date AND (p.replacementPackage IS NULL OR p.replacementPackage != 'spam/spam') + WHERE p.dumpedAt >= :date AND p.frozen IS NULL ") ->setParameters(['date' => $date]); @@ -72,7 +72,7 @@ public function getPackageNamesUpdatedSince(\DateTimeInterface $date): array public function getPackageNames(): array { $query = $this->getEntityManager() - ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.replacementPackage IS NULL OR p.replacementPackage != 'spam/spam'"); + ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.frozen IS NULL"); $names = $this->getPackageNamesForQuery($query); @@ -102,7 +102,7 @@ public function getProvidedNames(): array public function getPackageNamesByType(string $type): array { $query = $this->getEntityManager() - ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.type = :type AND (p.replacementPackage IS NULL OR p.replacementPackage != 'spam/spam')") + ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.type = :type AND p.frozen IS NULL") ->setParameters(['type' => $type]); return $this->getPackageNamesForQuery($query); @@ -114,7 +114,7 @@ public function getPackageNamesByType(string $type): array public function getPackageNamesByVendor(string $vendor): array { $query = $this->getEntityManager() - ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.vendor = :vendor AND (p.replacementPackage IS NULL OR p.replacementPackage != 'spam/spam')") + ->createQuery("SELECT p.name FROM App\Entity\Package p WHERE p.vendor = :vendor AND p.frozen IS NULL") ->setParameters(['vendor' => $vendor]); return $this->getPackageNamesForQuery($query); @@ -166,11 +166,10 @@ public function getPackagesWithFields(array $filters, array $fields): array $selector .= ', p.replacementPackage'; } - $where = '(p.replacementPackage IS NULL OR p.replacementPackage != :replacement)'; + $where = 'p.frozen IS NULL'; foreach ($filters as $filter => $val) { $where .= ' AND p.'.$filter.' = :'.$filter; } - $filters['replacement'] = "spam/spam"; $query = $this->getEntityManager() ->createQuery("SELECT p.name $selector FROM App\Entity\Package p WHERE $where ORDER BY p.name") ->setParameters($filters); @@ -218,6 +217,7 @@ public function getStalePackages(): array return $conn->fetchAllAssociative( 'SELECT p.id FROM package p WHERE p.abandoned = false + AND p.frozen IS NULL AND ( p.crawledAt IS NULL OR (p.autoUpdated = 0 AND p.crawledAt < :recent AND p.createdAt >= :yesterday) @@ -259,6 +259,7 @@ public function getStalePackagesForDumping(): array FROM package p LEFT JOIN download d ON (d.id = p.id AND d.type = 1) WHERE (p.dumpedAt IS NULL OR (p.dumpedAt <= p.crawledAt AND p.crawledAt < NOW())) + AND p.frozen IS NULL AND (d.total > 1000 OR d.lastUpdated > :date) ORDER BY p.crawledAt ASC ', ['date' => date('Y-m-d H:i:s', strtotime('-4months'))]); @@ -272,7 +273,7 @@ public function isPackageDumpableForV1(Package $package): bool SELECT p.id FROM package p LEFT JOIN download d ON (d.id = p.id AND d.type = 1) - WHERE p.id = :id + WHERE p.id = :id AND p.frozen IS NULL AND (d.total > 1000 OR d.lastUpdated > :date) ', ['id' => $package->getId(), 'date' => date('Y-m-d H:i:s', strtotime('-4months'))]); } @@ -284,7 +285,7 @@ public function getStalePackagesForDumpingV2(): array { $conn = $this->getEntityManager()->getConnection(); - return $conn->fetchFirstColumn('SELECT p.id FROM package p WHERE p.dumpedAtV2 IS NULL OR (p.dumpedAtV2 <= p.crawledAt AND p.crawledAt < NOW())'); + return $conn->fetchFirstColumn('SELECT p.id FROM package p WHERE (p.dumpedAtV2 IS NULL OR (p.dumpedAtV2 <= p.crawledAt AND p.crawledAt < NOW())) AND p.frozen IS NULL'); } /** @@ -414,7 +415,7 @@ public function getFilteredQueryBuilder(array $filters = [], bool $orderByName = $qb->leftJoin('v.tags', 't'); } - $qb->andWhere('(p.replacementPackage IS NULL OR p.replacementPackage != \'spam/spam\')'); + $qb->andWhere('p.frozen IS NULL'); $qb->orderBy('p.abandoned'); if (true === $orderByName) { @@ -466,7 +467,7 @@ public function markPackageSuspect(Package $package): void */ public function getSuspectPackageCount(): int { - $sql = 'SELECT COUNT(*) count FROM package p WHERE p.suspect IS NOT NULL AND (p.replacementPackage IS NULL OR p.replacementPackage != "spam/spam")'; + $sql = 'SELECT COUNT(*) count FROM package p WHERE p.suspect IS NOT NULL AND p.frozen IS NULL'; return max(0, (int) $this->getEntityManager()->getConnection()->fetchOne($sql)); } @@ -477,7 +478,7 @@ public function getSuspectPackageCount(): int public function getSuspectPackages(int $offset = 0, int $limit = 15): array { $sql = 'SELECT p.id, p.name, p.description, p.language, p.abandoned, p.replacementPackage - FROM package p WHERE p.suspect IS NOT NULL AND (p.replacementPackage IS NULL OR p.replacementPackage != "spam/spam") ORDER BY p.createdAt DESC LIMIT '.((int) $limit).' OFFSET '.((int) $offset); + FROM package p WHERE p.suspect IS NOT NULL AND p.frozen IS NULL ORDER BY p.createdAt DESC LIMIT '.((int) $limit).' OFFSET '.((int) $offset); return $this->getEntityManager()->getConnection()->fetchAllAssociative($sql); } diff --git a/src/Entity/VersionRepository.php b/src/Entity/VersionRepository.php index 6021563e2..01a95cb64 100644 --- a/src/Entity/VersionRepository.php +++ b/src/Entity/VersionRepository.php @@ -48,8 +48,8 @@ public function remove(Version $version): void $em = $this->getEntityManager(); $package = $version->getPackage(); $package->getVersions()->removeElement($version); - $package->setCrawledAt(new \DateTime); - $package->setUpdatedAt(new \DateTime); + $package->setCrawledAt(new \DateTimeImmutable()); + $package->setUpdatedAt(new \DateTimeImmutable()); $em->persist($package); $this->versionIdCache->deleteVersion($package, $version); diff --git a/src/Model/PackageManager.php b/src/Model/PackageManager.php index 599d064c8..06b9185a6 100644 --- a/src/Model/PackageManager.php +++ b/src/Model/PackageManager.php @@ -181,8 +181,8 @@ public function notifyUpdateFailure(Package $package, \Exception $e, ?string $de } // make sure the package crawl time is updated so we avoid retrying failing packages more often than working ones - if (!$package->getCrawledAt() || $package->getCrawledAt() < new \DateTime()) { - $package->setCrawledAt(new \DateTime); + if (!$package->getCrawledAt() || $package->getCrawledAt() < new \DateTimeImmutable()) { + $package->setCrawledAt(new \DateTimeImmutable()); } $this->doctrine->getManager()->flush(); diff --git a/src/Package/SymlinkDumper.php b/src/Package/SymlinkDumper.php index c60057342..87b4b0683 100644 --- a/src/Package/SymlinkDumper.php +++ b/src/Package/SymlinkDumper.php @@ -12,6 +12,7 @@ namespace App\Package; +use App\Entity\PackageFreezeReason; use Composer\Pcre\Preg; use Doctrine\DBAL\ArrayParameterType; use Seld\Signal\SignalHandler; @@ -193,9 +194,8 @@ public function dump(array $packageIds, bool $force = false, bool $verbose = fal // prepare packages in memory foreach ($packages as $package) { - // skip spam packages in the dumper in case we do a forced full dump and prevent them from being dumped for a little while - if ($package->isAbandoned() && $package->getReplacementPackage() === 'spam/spam') { - $dumpTimeUpdates['2100-01-01 00:00:00'][] = $package->getId(); + // skip spam packages in the dumper in case one appears due to a race condition + if ($package->isFrozen() && $package->getFreezeReason() === PackageFreezeReason::Spam) { continue; } diff --git a/src/Package/Updater.php b/src/Package/Updater.php index 534c177f9..cfa29be60 100644 --- a/src/Package/Updater.php +++ b/src/Package/Updater.php @@ -111,6 +111,10 @@ public function update(IOInterface $io, Config $config, Package $package, VcsRep throw new \RuntimeException('Driver could not be established for package '.$package->getName().' ('.$package->getRepository().')'); } + if ($package->isFrozen()) { + return $package; + } + $remoteId = null; if ($driver instanceof GitHubDriver) { $repoData = $driver->getRepoData(); @@ -303,12 +307,12 @@ public function update(IOInterface $io, Config $config, Package $package, VcsRep $io->writeError('Updated from '.$package->getRepository().' using ' . $driver::class . $usingDetails); // make sure the package exists in the package list if for some reason adding it on submit failed - if ($package->getReplacementPackage() !== 'spam/spam' && !$this->providerManager->packageExists($package->getName())) { + if (!$this->providerManager->packageExists($package->getName())) { $this->providerManager->insertPackage($package); } - $package->setUpdatedAt(new \DateTime); - $package->setCrawledAt(new \DateTime); + $package->setUpdatedAt(new \DateTimeImmutable()); + $package->setCrawledAt(new \DateTimeImmutable()); if ($flags & self::FORCE_DUMP) { $package->setDumpedAt(null); diff --git a/src/Package/V2Dumper.php b/src/Package/V2Dumper.php index 38a7609f9..6017a9513 100644 --- a/src/Package/V2Dumper.php +++ b/src/Package/V2Dumper.php @@ -12,6 +12,7 @@ namespace App\Package; +use App\Entity\PackageFreezeReason; use App\Entity\SecurityAdvisory; use Composer\Pcre\Preg; use Doctrine\DBAL\ArrayParameterType; @@ -111,9 +112,8 @@ public function dump(array $packageIds, bool $force = false, bool $verbose = fal // prepare packages in memory foreach ($packages as $package) { - // skip spam packages in the dumper in case we do a forced full dump and prevent them from being dumped for a little while - if ($package->isAbandoned() && $package->getReplacementPackage() === 'spam/spam') { - $dumpTimeUpdates['2100-01-01 00:00:00'][] = $package->getId(); + // skip spam packages in the dumper in case one appears due to a race condition + if ($package->isFrozen() && $package->getFreezeReason() === PackageFreezeReason::Spam) { continue; } diff --git a/src/Service/UpdaterWorker.php b/src/Service/UpdaterWorker.php index a150a9f7c..ffd8d8b55 100644 --- a/src/Service/UpdaterWorker.php +++ b/src/Service/UpdaterWorker.php @@ -12,6 +12,7 @@ namespace App\Service; +use App\Entity\PackageFreezeReason; use App\Entity\User; use App\SecurityAdvisory\FriendsOfPhpSecurityAdvisoriesSource; use Composer\Pcre\Preg; @@ -315,13 +316,17 @@ public function process(Job $job, SignalHandler $signal): array // detected a 404 so mark the package as gone and prevent updates for 1y if ($found404) { - $package->setCrawledAt($found404 === true ? new \DateTime('+1 year') : $found404); + if ($found404 === true) { + $package->freeze(PackageFreezeReason::Gone); + } else { + $package->setCrawledAt($found404); + } $this->getEM()->persist($package); $this->getEM()->flush(); return [ 'status' => Job::STATUS_PACKAGE_GONE, - 'message' => 'Update of '.$packageName.' failed, package appears to be 404/gone and has been marked as crawled for 1year', + 'message' => 'Update of '.$packageName.' failed, package appears to be 404/gone and has been marked frozen.', 'details' => '
'.$output.'
', 'exception' => $e, 'vendor' => $packageVendor, @@ -390,14 +395,26 @@ private function checkForDeadGitHubPackage(Package $package, string $repo, HttpD if ($e instanceof TransportException && in_array($e->getStatusCode(), [404, 409, 451], true)) { try { // check composer repo is visible to make sure it's not github or something else glitching - $httpDownloader->get('https://api.github.com/repos/composer/composer', ['retry-auth-failure' => false]); + $httpDownloader->get('https://api.github.com/repos/composer/composer/git/refs/heads', ['retry-auth-failure' => false]); // remove packages with very low downloads and that are 404 if ($this->downloadManager->getTotalDownloads($package) <= 100) { $this->packageManager->deletePackage($package); return [ 'status' => Job::STATUS_PACKAGE_DELETED, - 'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been deleted', + 'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been deleted.', + 'details' => '
'.$output.'
', + 'exception' => $e, + 'vendor' => $package->getVendor(), + ]; + } else { + $package->freeze(PackageFreezeReason::Gone); + $this->getEM()->persist($package); + $this->getEM()->flush(); + + return [ + 'status' => Job::STATUS_PACKAGE_GONE, + 'message' => 'Update of '.$package->getName().' failed, package appears to be 404/gone and has been marked frozen.', 'details' => '
'.$output.'
', 'exception' => $e, 'vendor' => $package->getVendor(), diff --git a/templates/macros.html.twig b/templates/macros.html.twig index f3e4f0afb..a6e40807d 100644 --- a/templates/macros.html.twig +++ b/templates/macros.html.twig @@ -1,6 +1,6 @@ {% macro listPackages(packages, paginate, showAutoUpdateWarning, meta) %}