From efa848ff68e2e35f04112af07574f10b85d09b32 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 08:37:15 -0700 Subject: [PATCH] various refactors including postgres portability improvements (#38) * refactor: use epoch seconds for timestamps * fix: use single query to filter for pulled time * refactor: use ResourceType instead of strings in most places * refactor: inline RevisionService into AbstractListService --- migrations/Version20241201012435.php | 10 +- .../Sync/Download/AbstractDownloadCommand.php | 19 +- .../Sync/Meta/AbstractMetaFetchCommand.php | 32 +-- src/Entity/SyncAsset.php | 5 +- src/Entity/SyncResource.php | 5 +- .../RevisionMetadataServiceInterface.php | 16 -- .../Interfaces/SubversionServiceInterface.php | 10 +- src/Services/List/AbstractListService.php | 185 ++++++++---------- src/Services/List/ListServiceInterface.php | 14 +- src/Services/List/PluginListService.php | 9 +- src/Services/List/ThemeListService.php | 11 +- .../Metadata/AbstractMetadataService.php | 132 +++++++------ .../Metadata/MetadataServiceInterface.php | 12 +- .../Metadata/PluginMetadataService.php | 6 +- .../Metadata/ThemeMetadataService.php | 6 +- src/Services/RevisionMetadataService.php | 63 ------ src/Services/SubversionService.php | 34 ++-- src/Utilities/ArrayUtil.php | 20 ++ tests/Stubs/MetadataServiceStub.php | 15 +- tests/Stubs/RevisionMetadataServiceStub.php | 30 --- tests/Stubs/SubversionServiceStub.php | 7 +- tests/Unit/Plugin/AbstractListServiceTest.php | 35 ---- 22 files changed, 280 insertions(+), 396 deletions(-) delete mode 100644 src/Services/Interfaces/RevisionMetadataServiceInterface.php delete mode 100644 src/Services/RevisionMetadataService.php delete mode 100644 tests/Stubs/RevisionMetadataServiceStub.php delete mode 100644 tests/Unit/Plugin/AbstractListServiceTest.php diff --git a/migrations/Version20241201012435.php b/migrations/Version20241201012435.php index b6d2fc0..da7ab23 100644 --- a/migrations/Version20241201012435.php +++ b/migrations/Version20241201012435.php @@ -26,8 +26,8 @@ public function up(Schema $schema): void status varchar(32) not null, version varchar(32), origin varchar(32) not null, - pulled timestamp(0) default current_timestamp not null, - updated timestamp(0) default current_timestamp not null, + pulled bigint not null, + updated bigint not null, metadata jsonb default null ) SQL, @@ -46,8 +46,8 @@ public function up(Schema $schema): void sync_id uuid not null references sync (id) on delete cascade, version varchar(32) not null, url text default null, - created timestamp(0) not null default current_timestamp, - processed timestamp(0) default null, + created bigint not null, + processed bigint default null, metadata jsonb default null ) SQL, @@ -61,7 +61,7 @@ public function up(Schema $schema): void create table revisions ( action varchar(255) not null, revision varchar(255) not null, - added_at timestamp(0) not null default current_timestamp + added bigint not null ) SQL, ); diff --git a/src/Commands/Sync/Download/AbstractDownloadCommand.php b/src/Commands/Sync/Download/AbstractDownloadCommand.php index 81107c3..8782252 100644 --- a/src/Commands/Sync/Download/AbstractDownloadCommand.php +++ b/src/Commands/Sync/Download/AbstractDownloadCommand.php @@ -46,17 +46,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->log->notice("Running command {$this->getName()}"); $this->startTimer(); - $force = $input->getOption('force'); - $numVersions = $input->getArgument('num-versions'); - $listing = $input->getOption($category); - $listing and $listing = StringUtil::explodeAndTrim($listing); + $force = $input->getOption('force'); + $numVersions = $input->getArgument('num-versions'); + $requested = array_fill_keys(StringUtil::explodeAndTrim($input->getOption($category) ?? ''), []); - $this->log->debug("Getting list of $category..."); - - if ($input->getOption('download-all')) { - $pending = $this->listService->getItems($listing); + if ($requested) { + $pending = $requested; + } elseif ($input->getOption('download-all')) { + $this->log->debug("Getting list of all $category..."); + $pending = $this->listService->getItems(); } else { - $pending = $this->listService->getUpdatedItems($listing); + $this->log->debug("Getting list of updated $category..."); + $pending = $this->listService->getUpdatedItems(); } $this->log->debug(count($pending) . " $category to download..."); diff --git a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php index 495ce2a..8949a82 100644 --- a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php +++ b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php @@ -39,12 +39,12 @@ abstract protected function makeRequest(string $slug): Request; protected function configure(): void { - $category = $this->resource->value . 's'; + $category = $this->resource->plural(); $this->setName("sync:meta:fetch:$category") ->setDescription("Fetches meta data of all new and changed $category") ->addOption( 'update-all', - 'u', + null, InputOption::VALUE_NONE, 'Update all metadata; otherwise, we only update what has changed' ) @@ -57,29 +57,37 @@ protected function configure(): void ->addOption( $category, null, - InputOption::VALUE_OPTIONAL, + InputOption::VALUE_REQUIRED, "List of $category (separated by commas) to explicitly update" ); } protected function execute(InputInterface $input, OutputInterface $output): int { - $category = $this->resource->value . 's'; + $category = $this->resource->plural(); $this->log->notice("Running command {$this->getName()}"); $this->startTimer(); - $items = StringUtil::explodeAndTrim($input->getOption($category) ?? ''); - $min_age = (int) $input->getOption('skip-newer-than-secs') ?: null; + $requested = array_fill_keys(StringUtil::explodeAndTrim($input->getOption($category) ?? ''), []); + $min_age = (int) $input->getOption('skip-newer-than-secs'); - $this->log->debug("Getting list of $category..."); - $toUpdate = $this->listService->getItems($items, $min_age); - $this->log->info(count($toUpdate) . " $category to download metadata for..."); + if ($requested) { + $toUpdate = $requested; + } else { + $this->log->debug("Getting list of $category..."); + $toUpdate = $this->listService->getItems(); + if ($min_age) { + $toUpdate = array_diff_key($toUpdate, $this->meta->getPulledAfter(time() - $min_age)); + } + } if (count($toUpdate) === 0) { $this->log->info('No metadata to download; exiting.'); return Command::SUCCESS; } + $this->log->info(count($toUpdate) . " $category to download metadata for..."); + $pool = $this->api->pool( requests: $this->generateRequests(array_keys($toUpdate)), concurrency: static::MAX_CONCURRENT_REQUESTS, @@ -90,7 +98,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $promise = $pool->send(); $promise->wait(); - if ($input->getOption($category)) { + if ($requested) { $this->log->debug("Not saving revision when --$category was specified"); } else { $revision = $this->listService->preserveRevision(); @@ -102,10 +110,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int } /** - * @param string[] $slugs + * @param (string|int)[] $slugs * @return Generator */ - protected function generateRequests(array $slugs): Generator + protected function generateRequests(iterable $slugs): Generator { foreach ($slugs as $slug) { yield $this->makeRequest((string) $slug); diff --git a/src/Entity/SyncAsset.php b/src/Entity/SyncAsset.php index 5ff00c5..c92a38d 100644 --- a/src/Entity/SyncAsset.php +++ b/src/Entity/SyncAsset.php @@ -5,7 +5,6 @@ namespace App\Entity; use App\Repository\SyncAssetRepository; -use DateTimeImmutable; use Doctrine\DBAL\Types\Types; use Doctrine\ORM\Mapping as ORM; use Symfony\Bridge\Doctrine\Types\UuidType; @@ -19,7 +18,7 @@ class SyncAsset { #[ORM\Column(nullable: true)] - public ?DateTimeImmutable $processed = null; // mutable! + public ?int $processed = null; // mutable! /** * @param array|null $metadata @@ -42,7 +41,7 @@ public function __construct( public readonly string $url, #[ORM\Column] - public readonly DateTimeImmutable $created, + public readonly int $created, #[ORM\Column(nullable: true)] public readonly ?array $metadata = null, diff --git a/src/Entity/SyncResource.php b/src/Entity/SyncResource.php index 53c1beb..a12aa3d 100644 --- a/src/Entity/SyncResource.php +++ b/src/Entity/SyncResource.php @@ -6,7 +6,6 @@ use App\Repository\SyncResourceRepository; use App\ResourceType; -use DateTimeImmutable; use Doctrine\Common\Collections\ArrayCollection; use Doctrine\Common\Collections\Collection; use Doctrine\DBAL\Types\Types; @@ -54,11 +53,11 @@ public function __construct( // when this record was synced #[ORM\Column] - public readonly DateTimeImmutable $pulled, + public readonly int $pulled, // last updated date in metadata #[ORM\Column] - public readonly ?DateTimeImmutable $updated, + public readonly ?int $updated, #[ORM\Column(nullable: true)] public readonly ?array $metadata = null, diff --git a/src/Services/Interfaces/RevisionMetadataServiceInterface.php b/src/Services/Interfaces/RevisionMetadataServiceInterface.php deleted file mode 100644 index 1764de2..0000000 --- a/src/Services/Interfaces/RevisionMetadataServiceInterface.php +++ /dev/null @@ -1,16 +0,0 @@ -, revision: int} */ - public function getUpdatedSlugs(string $type, int $prevRevision, int $lastRevision): array; + /** @return array{slugs: array, revision: int} */ + public function getUpdatedSlugs(ResourceType $type, int $prevRevision, int $lastRevision): array; - /** @return array{slugs: string[], revision: int} */ - public function scrapeSlugsFromIndex(string $type): array; + /** @return array{slugs: array, revision: int} */ + public function scrapeSlugsFromIndex(ResourceType $type): array; } diff --git a/src/Services/List/AbstractListService.php b/src/Services/List/AbstractListService.php index 4a290a2..5a813a0 100644 --- a/src/Services/List/AbstractListService.php +++ b/src/Services/List/AbstractListService.php @@ -4,142 +4,121 @@ namespace App\Services\List; -use App\Services\Interfaces\RevisionMetadataServiceInterface; +use App\ResourceType; use App\Services\Interfaces\SubversionServiceInterface; use App\Services\Metadata\MetadataServiceInterface; +use Doctrine\ORM\EntityManagerInterface; +use RuntimeException; -abstract readonly class AbstractListService implements ListServiceInterface +abstract class AbstractListService implements ListServiceInterface { + protected string $name; + public function __construct( protected SubversionServiceInterface $svn, protected MetadataServiceInterface $meta, - protected RevisionMetadataServiceInterface $revisions, - protected string $category, - ) {} - - /** - * @param string[] $filter - * @return array - */ - public function getItems(array $filter, ?int $min_age = null): array - { - $lastRevision = $this->revisions->getRevisionForAction($this->category); - $updates = $lastRevision - ? $this->getUpdatableItems($filter, $lastRevision) - : $this->getAllSubversionSlugs(); - return $this->filter($updates, $filter, $min_age); + protected EntityManagerInterface $em, + protected ResourceType $type, + protected string $origin = 'wp_org', + ) { + $this->name = "list-{$type->plural()}@$origin"; + $this->loadLatestRevisions(); } - public function preserveRevision(): string - { - return $this->revisions->preserveRevision($this->category); - } + //region Public API - /** - * @return array - * - * LOLPHP: should always return array but numeric keys like '1976' get forcibly cast to int when read. - */ - protected function getAllSubversionSlugs(): array + /** @return array */ + public function getItems(): array { - $result = $this->svn->scrapeSlugsFromIndex($this->category); - $this->revisions->setCurrentRevision($this->category, $result['revision']); - - // transform to [slug => [versions]] format - $out = []; - foreach ($result['slugs'] as $slug) { - $out[$slug] = []; - } - return $out; + $lastRevision = $this->getRevision(); + return $lastRevision ? $this->getUpdatableItems($lastRevision) : $this->getAllSubversionSlugs(); } - public function getUpdatedItems(?array $requested): array + /** @return array */ + public function getUpdatedItems(): array { - $revision = $this->revisions->getRevisionDateForAction($this->category); + $revision = $this->getRevisionDate(); if ($revision) { $revision = \Safe\date('Y-m-d', \Safe\strtotime($revision)); } - return $this->filter($this->meta->getOpenVersions($revision), $requested, null); + return $this->meta->getOpenVersions($revision); } - //region Protected API - - /** - * Reduces the items slated for update to only those specified in the filter. - * - * @param array $items - * @param array|null $filter - * @return array - */ - protected function filter(array $items, ?array $filter, ?int $min_age): array + public function preserveRevision(): string { - if (!$filter && !$min_age) { - return $items; + $name = $this->name; + if (!isset($this->currentRevision[$name])) { + throw new RuntimeException("No revision found for '$name'"); } + $revision = $this->currentRevision[$name]['revision']; + $this->em->getConnection()->insert( + 'revisions', + ['action' => $name, 'revision' => $revision, 'created' => time()] + ); + return (string) $revision; + } - $filtered = $filter ? [] : $items; + //endregion - foreach ($filter as $slug) { - if (array_key_exists($slug, $items)) { - $filtered[$slug] = $items[$slug]; - } - } + //region Protected API - $out = $min_age ? [] : $filtered; - if ($min_age) { - $cutoff = time() - $min_age; - foreach ($filtered as $slug => $value) { - $slug = (string) $slug; // LOLPHP: php will turn any numeric string key into an int - $timestamp = $this->meta->getPulledAsTimestamp($slug); - if ($timestamp === null || $timestamp <= $cutoff) { - $out[$slug] = $value; - } - } - } - return $out; + /** @return array */ + protected function getAllSubversionSlugs(): array + { + $result = $this->svn->scrapeSlugsFromIndex($this->type); + $this->setCurrentRevision($result['revision']); + return $result['slugs']; } - /** - * Takes the entire list of items, and adds any we have not seen before, - * plus merges items that we have explicitly queued for update. - * - * @param array $update - * @param string[] $requested - * @return array - */ - protected function addNewAndRequested(array $update, ?array $requested): array + /** @return array */ + protected function getUpdatableItems(string $lastRevision): array { - $allSlugs = $this->getAllSubversionSlugs(); - - foreach ($allSlugs as $slug => $versions) { - $slug = (string) $slug; - $status = $this->meta->getStatus($slug); - // Is this the first time we've seen the slug? - if (!$status) { - $update[$slug] = []; - } - if (in_array($slug, $requested, true)) { - $update[$slug] = []; - } - } - - return $update; + $output = $this->svn->getUpdatedSlugs($this->type, 0, (int) $lastRevision); // FIXME second arg should be prevRevision + $revision = $output['revision']; + $slugs = $output['slugs']; + $this->setCurrentRevision($revision); + $new = array_diff_key($this->getAllSubversionSlugs(), $this->meta->getAllSlugs()); + return [...$slugs, ...$new]; } - /** - * @param string[] $requested - * @return array - */ - protected function getUpdatableItems(?array $requested, string $lastRevision): array + //endregion + + //region RevisionService inlined + + /** @var array */ + private array $revisionData = []; + + /** @var array */ + private array $currentRevision = []; + + public function setCurrentRevision(int $revision): void { - $output = $this->svn->getUpdatedSlugs($this->category, 0, (int) $lastRevision); // FIXME second arg should be prevRevision + $this->currentRevision[$this->name] = ['revision' => $revision]; + } - $revision = $output['revision']; - $slugs = $output['slugs']; + public function getRevision(): ?string + { + return $this->revisionData[$this->name]['revision'] ?? null; + } - $this->revisions->setCurrentRevision($this->category, $revision); + public function getRevisionDate(): ?string + { + return $this->revisionData[$this->name]['added'] ?? null; + } - return $this->addNewAndRequested($slugs, $requested); + private function loadLatestRevisions(): void + { + $sql = <<em->getConnection()->fetchAllAssociative($sql) as $revision) { + $this->revisionData[$revision['action']] = [ + 'revision' => $revision['revision'], + 'added' => $revision['created'], + ]; + } } //endregion diff --git a/src/Services/List/ListServiceInterface.php b/src/Services/List/ListServiceInterface.php index d2c065f..9901aac 100644 --- a/src/Services/List/ListServiceInterface.php +++ b/src/Services/List/ListServiceInterface.php @@ -6,17 +6,11 @@ interface ListServiceInterface { - /** - * @param string[] $filter - * @return array - */ - public function getItems(array $filter, ?int $min_age = null): array; + /** @return array */ + public function getItems(): array; - /** - * @param string[] $requested - * @return array> - */ - public function getUpdatedItems(?array $requested): array; + /** @return array> */ + public function getUpdatedItems(): array; public function preserveRevision(): string; } diff --git a/src/Services/List/PluginListService.php b/src/Services/List/PluginListService.php index e276812..c232faa 100644 --- a/src/Services/List/PluginListService.php +++ b/src/Services/List/PluginListService.php @@ -4,14 +4,15 @@ namespace App\Services\List; +use App\ResourceType; use App\Services\Interfaces\SubversionServiceInterface; use App\Services\Metadata\PluginMetadataService; -use App\Services\RevisionMetadataService; +use Doctrine\ORM\EntityManagerInterface; -readonly class PluginListService extends AbstractListService +class PluginListService extends AbstractListService { - public function __construct(SubversionServiceInterface $svn, PluginMetadataService $meta, RevisionMetadataService $revisions) + public function __construct(SubversionServiceInterface $svn, PluginMetadataService $meta, EntityManagerInterface $em) { - parent::__construct($svn, $meta, $revisions, 'plugins'); + parent::__construct($svn, $meta, $em, ResourceType::Plugin); } } diff --git a/src/Services/List/ThemeListService.php b/src/Services/List/ThemeListService.php index 8170b95..c353db5 100644 --- a/src/Services/List/ThemeListService.php +++ b/src/Services/List/ThemeListService.php @@ -4,14 +4,15 @@ namespace App\Services\List; +use App\ResourceType; +use App\Services\Interfaces\SubversionServiceInterface; use App\Services\Metadata\ThemeMetadataService; -use App\Services\RevisionMetadataService; -use App\Services\SubversionService; +use Doctrine\ORM\EntityManagerInterface; -readonly class ThemeListService extends AbstractListService +class ThemeListService extends AbstractListService { - public function __construct(SubversionService $svn, ThemeMetadataService $meta, RevisionMetadataService $revisions) + public function __construct(SubversionServiceInterface $svn, ThemeMetadataService $meta, EntityManagerInterface $em) { - parent::__construct($svn, $meta, $revisions, 'themes'); + parent::__construct($svn, $meta, $em, ResourceType::Theme); } } diff --git a/src/Services/Metadata/AbstractMetadataService.php b/src/Services/Metadata/AbstractMetadataService.php index ad9b72a..53370f3 100644 --- a/src/Services/Metadata/AbstractMetadataService.php +++ b/src/Services/Metadata/AbstractMetadataService.php @@ -7,23 +7,27 @@ use App\ResourceType; use Doctrine\DBAL\ArrayParameterType; use Doctrine\DBAL\Connection; +use Doctrine\DBAL\Query\QueryBuilder; +use Doctrine\ORM\EntityManagerInterface; use Generator; use Psr\Log\LoggerInterface; use Ramsey\Uuid\Uuid; -use Safe\DateTimeImmutable; use function Safe\json_decode; use function Safe\json_encode; +use function Safe\strtotime; abstract readonly class AbstractMetadataService implements MetadataServiceInterface { public function __construct( - protected Connection $connection, + protected EntityManagerInterface $em, protected LoggerInterface $log, protected ResourceType $resource, protected string $origin = 'wp_org', ) {} + //region Public API + /** @param array $metadata */ public function save(array $metadata): void { @@ -33,7 +37,7 @@ public function save(array $metadata): void 'open' => $this->saveOpen(...), default => $this->saveError(...), }; - $this->connection->transactional(fn() => $method($metadata)); + $this->connection()->transactional(fn() => $method($metadata)); } /** @param array $metadata */ @@ -47,20 +51,21 @@ protected function saveOpen(array $metadata): void 'slug' => mb_substr($metadata['slug'], 0, 255), 'name' => mb_substr($metadata['name'], 0, 255), 'status' => 'open', - 'version' => $metadata['version'], + 'version' => mb_substr($metadata['version'], 0, 32), 'origin' => $this->origin, - 'updated' => \Safe\date('c', \Safe\strtotime($metadata['last_updated'])), - 'pulled' => \Safe\date('c'), + 'updated' => strtotime($metadata['last_updated'] ?? 'now'), + 'pulled' => time(), 'metadata' => $metadata, ]); $versions = $metadata['versions'] ?: [$metadata['version'] => $metadata['download_link']]; foreach ($versions as $version => $url) { - $this->connection->insert('sync_assets', [ + $this->connection()->insert('sync_assets', [ 'id' => Uuid::uuid7()->toString(), 'sync_id' => $id, 'version' => $version, 'url' => $url, + 'created' => time(), ]); } $this->log->debug( @@ -83,8 +88,8 @@ protected function saveError(array $metadata): void 'status' => $status, 'version' => null, 'origin' => $this->origin, - 'updated' => $metadata['closed_date'] ?? \Safe\date('c'), - 'pulled' => \Safe\date('c'), + 'updated' => strtotime($metadata['closed_date'] ?? 'now'), + 'pulled' => time(), 'metadata' => $metadata, ]; $this->insertSync($row); @@ -94,48 +99,40 @@ protected function saveError(array $metadata): void ); } - // These getters are more efficient than using ->fetch() - public function getStatus(string $slug): ?string - { - $sql = "select status from sync where slug = :slug and type = :type and origin = :origin"; - return $this->connection->fetchOne($sql, ['slug' => $slug, ...$this->stdArgs()]) ?: null; - } - - public function getPulledAsTimestamp(string $slug): ?int + /** @return array */ + public function getPulledAfter(int $timestamp): array { - $sql = "select unixepoch(pulled) from sync where slug = :slug and type = :type and origin = :origin"; - $pulled = $this->connection->fetchOne($sql, ['slug' => $slug, ...$this->stdArgs()]); - return (int) $pulled ?: null; + return $this->querySync() + ->select('slug', 'pulled') + ->andWhere('pulled > :timestamp') + ->setParameter('timestamp', $timestamp) + ->executeQuery() + ->fetchAllKeyValue(); } public function getDownloadUrl(string $slug, string $version): ?string { - $sql = <<<'SQL' - SELECT url - FROM sync_assets - JOIN sync ON sync.id = sync_assets.sync_id - WHERE sync.slug = :slug - AND sync.type = :type - AND sync.origin = :origin - AND sync_assets.version = :version - SQL; - $result = $this->connection->fetchAssociative($sql, ['slug' => $slug, 'version' => $version, ...$this->stdArgs()]); - return $result['url'] ?? null; + return $this->querySyncAssets($slug) + ->select('url') + ->where('sync_assets.version = :version') + ->setParameter('version', $version) + ->fetchOne(); } /** @return array [slug => [versions]] */ public function getOpenVersions(string $revDate = '1900-01-01'): array { - $sql = <<= :revDate + AND pulled >= :stamp AND sync.type = :type AND sync.origin = :origin SQL; - $result = $this->connection->fetchAllAssociative($sql, ['revDate' => $revDate, ...$this->stdArgs()]); + $result = $this->connection()->fetchAllAssociative($sql, ['stamp' => $stamp, ...$this->stdArgs()]); $out = []; foreach ($result as $row) { @@ -144,23 +141,26 @@ public function getOpenVersions(string $revDate = '1900-01-01'): array return $out; } + public function getAllSlugs(): array + { + return $this->querySync()->select('slug')->executeQuery()->fetchFirstColumn() ?: []; + } + public function markProcessed(string $slug, string $version): void { $sql = <<<'SQL' UPDATE sync_assets - SET processed = current_timestamp + SET processed = :stamp WHERE version = :version AND sync_id = (SELECT id FROM sync WHERE slug = :slug AND type = :type AND origin = :origin) SQL; - $this->connection->executeQuery($sql, ['slug' => $slug, 'version' => $version, ...$this->stdArgs()]); - // Most things that call this already log in some other way - // $this->log->debug("Processed $slug", ['type' => $this->resource->value, 'slug' => $slug, 'version' => $version]); + $this->connection()->executeQuery($sql, ['stamp' => time(), 'slug' => $slug, 'version' => $version, ...$this->stdArgs()]); } public function exportAllMetadata(): Generator { $sql = "select * from sync where status in ('open', 'closed') and type = :type and origin = :origin"; - $rows = $this->connection->executeQuery($sql, $this->stdArgs()); + $rows = $this->connection()->executeQuery($sql, $this->stdArgs()); while ($row = $rows->fetchAssociative()) { $metadata = json_decode($row['metadata'], true); unset($row['metadata']); @@ -186,7 +186,7 @@ public function getUnprocessedVersions(string $slug, array $versions): array and sync.origin = :origin SQL; - $results = $this->connection->executeQuery( + $results = $this->connection()->executeQuery( $sql, ['slug' => $slug, 'versions' => $versions, ...$this->stdArgs()], ['versions' => ArrayParameterType::STRING] @@ -194,39 +194,51 @@ public function getUnprocessedVersions(string $slug, array $versions): array return $results->fetchFirstColumn(); } + //endregion + + //region Protected/Private API + /** @return array{type: string, origin: string} */ protected function stdArgs(): array { return ['type' => $this->resource->value, 'origin' => $this->origin]; } - // keep fetch protected so we don't expose the raw db schema publicly - /** @return array>|null */ - protected function fetch(string $slug): ?array + protected function querySync(): QueryBuilder { - $sql = "select * from sync where slug = :slug and type = :type and origin = :origin"; - $params = ['slug' => $slug, ...$this->stdArgs()]; - $item = $this->connection->fetchAssociative($sql, $params); - return [ - 'id' => $item['id'], - 'type' => $item['type'], - 'slug' => $item['slug'], - 'name' => $item['name'], - 'status' => $item['status'], - 'version' => $item['version'], - 'origin' => $item['origin'], - 'updated' => new DateTimeImmutable($item['updated']), - 'pulled' => new DateTimeImmutable($item['pulled']), - 'metadata' => json_decode($item['metadata'] ?? 'null'), - ]; + return $this->connection()->createQueryBuilder() + ->select('*') + ->from('sync') + ->andWhere('type = :type') + ->andWhere('origin = :origin') + ->setParameter('type', $this->resource->value) + ->setParameter('origin', $this->origin); + } + + protected function querySyncAssets(string $slug): QueryBuilder + { + return $this->connection()->createQueryBuilder() + ->select('*') + ->from('sync_assets') + ->andWhere('sync_assets.sync_id = (SELECT id FROM sync WHERE slug = :slug AND type = :type AND origin = :origin)') + ->setParameter('slug', $slug) + ->setParameter('type', $this->resource->value) + ->setParameter('origin', $this->origin); } - /** @param array{slug: string, metadata: mixed} $args */ + /** @param array $args */ protected function insertSync(array $args): void { $args['metadata'] = json_encode($args['metadata']); - $conn = $this->connection; + $conn = $this->connection(); $conn->delete('sync', ['slug' => $args['slug'], ...$this->stdArgs()]); $conn->insert('sync', $args); } + + private function connection(): Connection + { + return $this->em->getConnection(); + } + + //endregion } diff --git a/src/Services/Metadata/MetadataServiceInterface.php b/src/Services/Metadata/MetadataServiceInterface.php index fbc0bdc..3b6436d 100644 --- a/src/Services/Metadata/MetadataServiceInterface.php +++ b/src/Services/Metadata/MetadataServiceInterface.php @@ -21,12 +21,14 @@ public function exportAllMetadata(): Generator; /** @param array $metadata */ public function save(array $metadata): void; - public function getStatus(string $slug): ?string; - - public function getPulledAsTimestamp(string $slug): ?int; - - /** @return array */ + /** @return array */ public function getOpenVersions(string $revDate = '1900-01-01'): array; public function markProcessed(string $slug, string $version): void; + + /** @return array */ + public function getPulledAfter(int $timestamp): array; + + /** @return array */ + public function getAllSlugs(): array; } diff --git a/src/Services/Metadata/PluginMetadataService.php b/src/Services/Metadata/PluginMetadataService.php index 46dff72..fdf17cb 100644 --- a/src/Services/Metadata/PluginMetadataService.php +++ b/src/Services/Metadata/PluginMetadataService.php @@ -5,13 +5,13 @@ namespace App\Services\Metadata; use App\ResourceType; -use Doctrine\DBAL\Connection; +use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; readonly class PluginMetadataService extends AbstractMetadataService { - public function __construct(Connection $connection, LoggerInterface $log) + public function __construct(EntityManagerInterface $em, LoggerInterface $log) { - parent::__construct($connection, $log, ResourceType::Plugin); + parent::__construct($em, $log, ResourceType::Plugin); } } diff --git a/src/Services/Metadata/ThemeMetadataService.php b/src/Services/Metadata/ThemeMetadataService.php index 356ca2b..64a8aed 100644 --- a/src/Services/Metadata/ThemeMetadataService.php +++ b/src/Services/Metadata/ThemeMetadataService.php @@ -5,13 +5,13 @@ namespace App\Services\Metadata; use App\ResourceType; -use Doctrine\DBAL\Connection; +use Doctrine\ORM\EntityManagerInterface; use Psr\Log\LoggerInterface; readonly class ThemeMetadataService extends AbstractMetadataService { - public function __construct(Connection $connection, LoggerInterface $log) + public function __construct(EntityManagerInterface $em, LoggerInterface $log) { - parent::__construct($connection, $log, ResourceType::Theme); + parent::__construct($em, $log, ResourceType::Theme); } } diff --git a/src/Services/RevisionMetadataService.php b/src/Services/RevisionMetadataService.php deleted file mode 100644 index 72f456b..0000000 --- a/src/Services/RevisionMetadataService.php +++ /dev/null @@ -1,63 +0,0 @@ - */ - private array $revisionData = []; - - /** @var array */ - private array $currentRevision = []; - - public function __construct(private Connection $connection) - { - $this->loadLatestRevisions(); - } - - public function setCurrentRevision(string $action, int $revision): void - { - $this->currentRevision[$action] = ['revision' => $revision]; - } - - public function preserveRevision(string $action): string - { - if (!isset($this->currentRevision[$action])) { - throw new RuntimeException('You did not specify a revision for action ' . $action); - } - $revision = $this->currentRevision[$action]['revision']; - $this->connection->insert('revisions', ['action' => $action, 'revision' => $revision]); - return (string) $revision; - } - - public function getRevisionForAction(string $action): ?string - { - return $this->revisionData[$action]['revision'] ?? null; - } - - public function getRevisionDateForAction(string $action): ?string - { - return $this->revisionData[$action]['added'] ?? null; - } - - private function loadLatestRevisions(): void - { - $sql = <<connection->fetchAllAssociative($sql) as $revision) { - $this->revisionData[$revision['action']] = [ - 'revision' => $revision['revision'], - 'added' => $revision['added_at'], - ]; - } - } -} diff --git a/src/Services/SubversionService.php b/src/Services/SubversionService.php index bbe6bec..240dec1 100644 --- a/src/Services/SubversionService.php +++ b/src/Services/SubversionService.php @@ -4,6 +4,7 @@ namespace App\Services; +use App\ResourceType; use App\Services\Interfaces\SubversionServiceInterface; use App\Utilities\RegexUtil; use GuzzleHttp\Client as GuzzleClient; @@ -14,21 +15,22 @@ class SubversionService implements SubversionServiceInterface { public function __construct(private readonly GuzzleClient $guzzle) {} - /** @return array{slugs: array, revision: int} */ - public function getUpdatedSlugs(string $type, int $prevRevision, int $lastRevision): array + /** @return array{slugs: array, revision: int} */ + public function getUpdatedSlugs(ResourceType $type, int $prevRevision, int $lastRevision): array { if ($prevRevision === $lastRevision) { return ['revision' => $lastRevision, 'slugs' => []]; } - $command = ['svn', 'log', '-v', '-q', '--xml', "https://$type.svn.wordpress.org", "-r", "$lastRevision:HEAD"]; + $url = "https://{$type->plural()}.svn.wordpress.org"; + $command = ['svn', 'log', '-v', '-q', '--xml', $url, "-r", "$lastRevision:HEAD"]; // example: svn log -v -q --xml https://plugins.svn.wordpress.org -r 3179185:HEAD $process = new Process($command); $process->run(); if (!$process->isSuccessful()) { - throw new RuntimeException("Unable to get list of $type to update: {$process->getErrorOutput()}"); + throw new RuntimeException("Unable to get list of {$type->plural()} to update: {$process->getErrorOutput()}"); } $output = \Safe\simplexml_load_string($process->getOutput()); @@ -42,28 +44,30 @@ public function getUpdatedSlugs(string $type, int $prevRevision, int $lastRevisi $path = (string) $entry->paths->path[0]; $matches = RegexUtil::match('#/([A-z\-_]+)/#', $path); if ($matches) { - $item = trim($matches[1]); - $slugs[$item] = []; + $slug = trim($matches[1]); + $slugs[$slug] = []; } } - return ['revision' => $revision, 'slugs' => $slugs]; + return ['slugs' => $slugs, 'revision' => $revision]; } - /** @return array{slugs: string[], revision: int} */ - public function scrapeSlugsFromIndex(string $type): array + /** @return array{slugs: array, revision: int} */ + public function scrapeSlugsFromIndex(ResourceType $type): array { $html = $this->guzzle - ->get("https://$type.svn.wordpress.org/") + ->get("https://{$type->plural()}.svn.wordpress.org/") ->getBody() ->getContents(); - [, $slugs] = RegexUtil::matchAll('#
  • ([^/]+)/
    • #', $html); + [, $rawslugs] = RegexUtil::matchAll('#
  • ([^/]+)/
    • #', $html); [, $revision] = RegexUtil::match('/Revision (\d+):/', $html); + $revision = (int) $revision or throw new RuntimeException("Unable to get revision from {$type->plural()} index"); - $slugs = array_map(urldecode(...), $slugs); - // $slugs = array_map(fn ($slug) => (string) urldecode($slug), $slugs); - - return ['slugs' => $slugs, 'revision' => (int) $revision]; + $slugs = []; + foreach ($rawslugs as $slug) { + $slugs[urldecode($slug)] = []; + } + return ['slugs' => $slugs, 'revision' => $revision]; } } diff --git a/src/Utilities/ArrayUtil.php b/src/Utilities/ArrayUtil.php index 84f6a73..43c6ea4 100644 --- a/src/Utilities/ArrayUtil.php +++ b/src/Utilities/ArrayUtil.php @@ -28,6 +28,26 @@ public static function fromEntries(iterable $entries): array return $assoc; } + /** + * @param array $arr + * @param string[] $keys + * @return array + */ + public static function onlyKeys(array $arr, array $keys): array + { + return array_intersect_key($arr, array_flip($keys)); + } + + /** + * @param array $arr + * @param string[] $keys + * @return array + */ + public static function withoutKeys(array $arr, array $keys): array + { + return array_diff_key($arr, array_flip($keys)); + } + private function __construct() { // not instantiable diff --git a/tests/Stubs/MetadataServiceStub.php b/tests/Stubs/MetadataServiceStub.php index 6ce6845..0d6d973 100644 --- a/tests/Stubs/MetadataServiceStub.php +++ b/tests/Stubs/MetadataServiceStub.php @@ -34,11 +34,6 @@ public function getStatus(string $slug): ?string return 'open'; } - public function getPulledAsTimestamp(string $slug): ?int - { - return 12345; - } - public function getOpenVersions(string $revDate = '1900-01-01'): array { return []; @@ -48,4 +43,14 @@ public function markProcessed(string $slug, string $version): void { // nothingness } + + public function getPulledAfter(int $timestamp): array + { + return []; + } + + public function getAllSlugs(): array + { + return []; + } } diff --git a/tests/Stubs/RevisionMetadataServiceStub.php b/tests/Stubs/RevisionMetadataServiceStub.php deleted file mode 100644 index dbb0eee..0000000 --- a/tests/Stubs/RevisionMetadataServiceStub.php +++ /dev/null @@ -1,30 +0,0 @@ - 123, @@ -16,11 +17,11 @@ public function getUpdatedSlugs(string $type, int $prevRevision, int $lastRevisi ]; } - public function scrapeSlugsFromIndex(string $type): array + public function scrapeSlugsFromIndex(ResourceType $type): array { return [ 'revision' => 123, - 'slugs' => ['foo', 'bar', 'baz'], + 'slugs' => ['foo' => [], 'bar' => [], 'baz' => []], ]; } } diff --git a/tests/Unit/Plugin/AbstractListServiceTest.php b/tests/Unit/Plugin/AbstractListServiceTest.php deleted file mode 100644 index 82c435f..0000000 --- a/tests/Unit/Plugin/AbstractListServiceTest.php +++ /dev/null @@ -1,35 +0,0 @@ -sut = new readonly class extends AbstractListService { - public function __construct() - { - $svn = new SubversionServiceStub(); - $meta = new MetadataServiceStub(); - $rev = new RevisionMetadataServiceStub(); - parent::__construct($svn, $meta, $rev, 'stuff'); - } - }; - } - - public function testStuff(): void - { - $items = $this->sut->getItems([]); - $this->assertEquals(['foo', 'bar', 'baz'], array_keys($items)); - } -}