various refactors including postgres portability improvements (#38)

* refactor: use epoch seconds for timestamps * fix: use single query to filter for pulled time * refactor: use ResourceType instead of strings in most places * refactor: inline RevisionService into AbstractListService
aspirepress · Dec 6, 2024 · efa848f · efa848f
1 parent 565d1d1
commit efa848f
Show file tree

Hide file tree

Showing 22 changed files with 280 additions and 396 deletions.
diff --git a/migrations/Version20241201012435.php b/migrations/Version20241201012435.php
@@ -26,8 +26,8 @@ public function up(Schema $schema): void
                       status varchar(32) not null,
                       version varchar(32),
                       origin varchar(32) not null,
-                      pulled timestamp(0) default current_timestamp not null,
-                      updated timestamp(0) default current_timestamp not null,
+                      pulled bigint not null,
+                      updated bigint not null,
                       metadata jsonb default null
                   )
                 SQL,
@@ -46,8 +46,8 @@ public function up(Schema $schema): void
                           sync_id uuid not null references sync (id) on delete cascade,
                           version varchar(32) not null,
                           url text default null,
-                          created timestamp(0) not null default current_timestamp,
-                          processed timestamp(0) default null,
+                          created bigint not null,
+                          processed bigint default null,
                           metadata jsonb default null
                         )
                 SQL,
@@ -61,7 +61,7 @@ public function up(Schema $schema): void
                 create table revisions (
                     action varchar(255) not null,
                     revision varchar(255) not null,
-                    added_at timestamp(0) not null default current_timestamp
+                    added bigint not null
                 )
                 SQL,
         );

diff --git a/src/Commands/Sync/Download/AbstractDownloadCommand.php b/src/Commands/Sync/Download/AbstractDownloadCommand.php
@@ -46,17 +46,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $this->log->notice("Running command {$this->getName()}");
         $this->startTimer();
-        $force                = $input->getOption('force');
-        $numVersions          = $input->getArgument('num-versions');
-        $listing              = $input->getOption($category);
-        $listing and $listing = StringUtil::explodeAndTrim($listing);
+        $force       = $input->getOption('force');
+        $numVersions = $input->getArgument('num-versions');
+        $requested   = array_fill_keys(StringUtil::explodeAndTrim($input->getOption($category) ?? ''), []);
 
-        $this->log->debug("Getting list of $category...");
-
-        if ($input->getOption('download-all')) {
-            $pending = $this->listService->getItems($listing);
+        if ($requested) {
+            $pending = $requested;
+        } elseif ($input->getOption('download-all')) {
+            $this->log->debug("Getting list of all $category...");
+            $pending = $this->listService->getItems();
         } else {
-            $pending = $this->listService->getUpdatedItems($listing);
+            $this->log->debug("Getting list of updated $category...");
+            $pending = $this->listService->getUpdatedItems();
         }
 
         $this->log->debug(count($pending) . " $category to download...");

diff --git a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php
@@ -39,12 +39,12 @@ abstract protected function makeRequest(string $slug): Request;
 
     protected function configure(): void
     {
-        $category = $this->resource->value . 's';
+        $category = $this->resource->plural();
         $this->setName("sync:meta:fetch:$category")
             ->setDescription("Fetches meta data of all new and changed $category")
             ->addOption(
                 'update-all',
-                'u',
+                null,
                 InputOption::VALUE_NONE,
                 'Update all metadata; otherwise, we only update what has changed'
             )
@@ -57,29 +57,37 @@ protected function configure(): void
             ->addOption(
                 $category,
                 null,
-                InputOption::VALUE_OPTIONAL,
+                InputOption::VALUE_REQUIRED,
                 "List of $category (separated by commas) to explicitly update"
             );
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
-        $category = $this->resource->value . 's';
+        $category = $this->resource->plural();
         $this->log->notice("Running command {$this->getName()}");
         $this->startTimer();
 
-        $items   = StringUtil::explodeAndTrim($input->getOption($category) ?? '');
-        $min_age = (int) $input->getOption('skip-newer-than-secs') ?: null;
+        $requested = array_fill_keys(StringUtil::explodeAndTrim($input->getOption($category) ?? ''), []);
+        $min_age = (int) $input->getOption('skip-newer-than-secs');
 
-        $this->log->debug("Getting list of $category...");
-        $toUpdate = $this->listService->getItems($items, $min_age);
-        $this->log->info(count($toUpdate) . " $category to download metadata for...");
+        if ($requested) {
+            $toUpdate = $requested;
+        } else {
+            $this->log->debug("Getting list of $category...");
+            $toUpdate = $this->listService->getItems();
+            if ($min_age) {
+                $toUpdate = array_diff_key($toUpdate, $this->meta->getPulledAfter(time() - $min_age));
+            }
+        }
 
         if (count($toUpdate) === 0) {
             $this->log->info('No metadata to download; exiting.');
             return Command::SUCCESS;
         }
 
+        $this->log->info(count($toUpdate) . " $category to download metadata for...");
+
         $pool = $this->api->pool(
             requests: $this->generateRequests(array_keys($toUpdate)),
             concurrency: static::MAX_CONCURRENT_REQUESTS,
@@ -90,7 +98,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $promise = $pool->send();
         $promise->wait();
 
-        if ($input->getOption($category)) {
+        if ($requested) {
             $this->log->debug("Not saving revision when --$category was specified");
         } else {
             $revision = $this->listService->preserveRevision();
@@ -102,10 +110,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     }
 
     /**
-     * @param string[] $slugs
+     * @param (string|int)[] $slugs
      * @return Generator<Request>
      */
-    protected function generateRequests(array $slugs): Generator
+    protected function generateRequests(iterable $slugs): Generator
     {
         foreach ($slugs as $slug) {
             yield $this->makeRequest((string) $slug);

diff --git a/src/Entity/SyncAsset.php b/src/Entity/SyncAsset.php
@@ -5,7 +5,6 @@
 namespace App\Entity;
 
 use App\Repository\SyncAssetRepository;
-use DateTimeImmutable;
 use Doctrine\DBAL\Types\Types;
 use Doctrine\ORM\Mapping as ORM;
 use Symfony\Bridge\Doctrine\Types\UuidType;
@@ -19,7 +18,7 @@
 class SyncAsset
 {
     #[ORM\Column(nullable: true)]
-    public ?DateTimeImmutable $processed = null;  // mutable!
+    public ?int $processed = null;  // mutable!
 
     /**
      * @param array<string, mixed>|null $metadata
@@ -42,7 +41,7 @@ public function __construct(
         public readonly string $url,
 
         #[ORM\Column]
-        public readonly DateTimeImmutable $created,
+        public readonly int $created,
 
         #[ORM\Column(nullable: true)]
         public readonly ?array $metadata = null,

diff --git a/src/Entity/SyncResource.php b/src/Entity/SyncResource.php
@@ -6,7 +6,6 @@
 
 use App\Repository\SyncResourceRepository;
 use App\ResourceType;
-use DateTimeImmutable;
 use Doctrine\Common\Collections\ArrayCollection;
 use Doctrine\Common\Collections\Collection;
 use Doctrine\DBAL\Types\Types;
@@ -54,11 +53,11 @@ public function __construct(
 
         // when this record was synced
         #[ORM\Column]
-        public readonly DateTimeImmutable $pulled,
+        public readonly int $pulled,
 
         // last updated date in metadata
         #[ORM\Column]
-        public readonly ?DateTimeImmutable $updated,
+        public readonly ?int $updated,
 
         #[ORM\Column(nullable: true)]
         public readonly ?array $metadata = null,

diff --git a/src/Services/Interfaces/RevisionMetadataServiceInterface.php b/src/Services/Interfaces/RevisionMetadataServiceInterface.php
diff --git a/src/Services/Interfaces/SubversionServiceInterface.php b/src/Services/Interfaces/SubversionServiceInterface.php
@@ -4,11 +4,13 @@
 
 namespace App\Services\Interfaces;
 
+use App\ResourceType;
+
 interface SubversionServiceInterface
 {
-    /** @return array{slugs: array<string, string[]>, revision: int} */
-    public function getUpdatedSlugs(string $type, int $prevRevision, int $lastRevision): array;
+    /** @return array{slugs: array<string|int, array{}>, revision: int} */
+    public function getUpdatedSlugs(ResourceType $type, int $prevRevision, int $lastRevision): array;
 
-    /** @return array{slugs: string[], revision: int} */
-    public function scrapeSlugsFromIndex(string $type): array;
+    /** @return array{slugs: array<string|int, array{}>, revision: int} */
+    public function scrapeSlugsFromIndex(ResourceType $type): array;
 }