From 3c93cf35b26d3405368cc3d4f42385744830756c Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Fri, 2 Feb 2024 17:39:05 +0000 Subject: [PATCH 01/18] When there's more pages, warm them --- src/Console/Commands/StaticWarm.php | 17 +++++++++++++++++ src/StaticCaching/Middleware/Cache.php | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/src/Console/Commands/StaticWarm.php b/src/Console/Commands/StaticWarm.php index 2a654329f2..9cebc840fa 100644 --- a/src/Console/Commands/StaticWarm.php +++ b/src/Console/Commands/StaticWarm.php @@ -19,6 +19,7 @@ use Statamic\Facades\URL; use Statamic\Http\Controllers\FrontendController; use Statamic\StaticCaching\Cacher as StaticCacher; +use Statamic\Support\Arr; use Statamic\Support\Str; use Statamic\Taxonomies\LocalizedTerm; use Statamic\Taxonomies\Taxonomy; @@ -117,6 +118,22 @@ private function concurrency(): int public function outputSuccessLine(Response $response, $index): void { $this->checkLine($this->getRelativeUri($index)); + + if ($response->hasHeader('Statamic-Pagination-Next')) { + $nextPageUrl = Arr::first($response->getHeader('Statamic-Pagination-Next')); + + // TODO: find a better way of warming this URL + $request = new Request('GET', $nextPageUrl); + $client = new Client([ + 'verify' => $this->shouldVerifySsl(), + 'auth' => $this->option('user') && $this->option('password') + ? [$this->option('user'), $this->option('password')] + : null, + ]); + + $response = $client->send($request); + $this->outputSuccessLine($response, $index); + } } public function outputFailureLine($exception, $index): void diff --git a/src/StaticCaching/Middleware/Cache.php b/src/StaticCaching/Middleware/Cache.php index 7b889214ab..d798890323 100644 --- a/src/StaticCaching/Middleware/Cache.php +++ b/src/StaticCaching/Middleware/Cache.php @@ -6,6 +6,7 @@ use Illuminate\Http\Response; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; +use Statamic\Facades\Blink; use Statamic\Facades\File; use Statamic\Statamic; use Statamic\StaticCaching\Cacher; @@ -65,6 +66,12 @@ public function handle($request, Closure $next) $this->makeReplacementsAndCacheResponse($request, $response); $this->nocache->write(); + + if ($paginator = Blink::get('tag-paginator')) { + if ($paginator->hasMorePages() > 1) { + $response->headers->set('Statamic-Pagination-Next', $paginator->nextPageUrl()); + } + } } elseif (! $response->isRedirect()) { $this->makeReplacements($response); } From 0bc87ea34e342b4f1c9c776042d3ff7e1921def4 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Thu, 8 Feb 2024 19:56:16 +0000 Subject: [PATCH 02/18] wip --- src/StaticCaching/Middleware/Cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/StaticCaching/Middleware/Cache.php b/src/StaticCaching/Middleware/Cache.php index d798890323..1b4139e56e 100644 --- a/src/StaticCaching/Middleware/Cache.php +++ b/src/StaticCaching/Middleware/Cache.php @@ -68,7 +68,7 @@ public function handle($request, Closure $next) $this->nocache->write(); if ($paginator = Blink::get('tag-paginator')) { - if ($paginator->hasMorePages() > 1) { + if ($paginator->hasMorePages()) { $response->headers->set('Statamic-Pagination-Next', $paginator->nextPageUrl()); } } From d2fdfd705aee50f357c7a3a922608f69b7670857 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Thu, 8 Feb 2024 19:56:28 +0000 Subject: [PATCH 03/18] make things work another way --- src/Console/Commands/StaticWarm.php | 52 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Console/Commands/StaticWarm.php b/src/Console/Commands/StaticWarm.php index 9cebc840fa..947b002a3e 100644 --- a/src/Console/Commands/StaticWarm.php +++ b/src/Console/Commands/StaticWarm.php @@ -42,6 +42,8 @@ class StaticWarm extends Command private $uris; + private $additionalUris = []; + public function handle() { if (! config('statamic.static_caching.strategy')) { @@ -59,19 +61,10 @@ public function handle() $this->comment('Please wait. This may take a while if you have a lot of content.'); - $this->warm(); - $this->output->newLine(); - $this->info($this->shouldQueue - ? 'All requests to warm the static cache have been added to the queue.' - : 'The static cache has been warmed.' - ); - - return 0; - } + $this->line('Compiling URLs...'); + $this->output->newLine(); - private function warm(): void - { $client = new Client([ 'verify' => $this->shouldVerifySsl(), 'auth' => $this->option('user') && $this->option('password') @@ -79,13 +72,30 @@ private function warm(): void : null, ]); - $this->output->newLine(); - $this->line('Compiling URLs...'); + $this->warm($client, $this->requests()); - $requests = $this->requests(); + // TODO: Figure out if this works with queued warming.. we may need to figure out a different way if not. + while (count($this->additionalUris) > 0) { + $additionalRequests = collect($this->additionalUris) + ->map(fn ($uri) => new Request('GET', $uri)) + ->all(); + + $this->additionalUris = []; + + $this->warm($client, $additionalRequests); + } $this->output->newLine(); + $this->info($this->shouldQueue + ? 'All requests to warm the static cache have been added to the queue.' + : 'The static cache has been warmed.' + ); + return 0; + } + + private function warm(Client $client, array $requests): void + { if ($this->shouldQueue) { $queue = config('statamic.static_caching.warm_queue'); $this->line(sprintf('Adding %s requests onto %squeue...', count($requests), $queue ? $queue.' ' : '')); @@ -120,19 +130,7 @@ public function outputSuccessLine(Response $response, $index): void $this->checkLine($this->getRelativeUri($index)); if ($response->hasHeader('Statamic-Pagination-Next')) { - $nextPageUrl = Arr::first($response->getHeader('Statamic-Pagination-Next')); - - // TODO: find a better way of warming this URL - $request = new Request('GET', $nextPageUrl); - $client = new Client([ - 'verify' => $this->shouldVerifySsl(), - 'auth' => $this->option('user') && $this->option('password') - ? [$this->option('user'), $this->option('password')] - : null, - ]); - - $response = $client->send($request); - $this->outputSuccessLine($response, $index); + $this->additionalUris[] = Arr::first($response->getHeader('Statamic-Pagination-Next')); } } From a697658b72589d6816b71064e50350149fed5924 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Fri, 9 Feb 2024 16:35:23 +0000 Subject: [PATCH 04/18] When `ignore_query_strings` is enabled, keep `page` parameter --- src/StaticCaching/Cachers/AbstractCacher.php | 13 ++++++++++++- src/StaticCaching/Cachers/FileCacher.php | 9 ++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/StaticCaching/Cachers/AbstractCacher.php b/src/StaticCaching/Cachers/AbstractCacher.php index 1fced8bf7e..90e7560175 100644 --- a/src/StaticCaching/Cachers/AbstractCacher.php +++ b/src/StaticCaching/Cachers/AbstractCacher.php @@ -8,6 +8,7 @@ use Statamic\Facades\Site; use Statamic\StaticCaching\Cacher; use Statamic\StaticCaching\UrlExcluder; +use Statamic\Support\Arr; use Statamic\Support\Str; abstract class AbstractCacher implements Cacher @@ -138,8 +139,18 @@ public function getUrl(Request $request) { $url = $request->getUri(); + // When ignore_query_strings is enabled, strip out all query params except for `page`. if ($this->config('ignore_query_strings')) { - $url = explode('?', $url)[0]; + $urlWithoutParams = Arr::get(explode('?', $url), 0); + $queryParams = Arr::get(explode('?', $url), 1); + + $url = $urlWithoutParams; + + if ($queryParams) { + $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) { + return Str::startsWith($param, 'page='); + })->implode('&'); + } } return $url; diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index 11a9d3afa5..75604c0b3d 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -165,7 +165,14 @@ public function getFilePath($url, $site = null) $urlParts = parse_url($url); $pathParts = pathinfo($urlParts['path']); $slug = $pathParts['basename']; - $query = $this->config('ignore_query_strings') ? '' : Arr::get($urlParts, 'query', ''); + $query = Arr::get($urlParts, 'query', ''); + + // When ignore_query_strings is enabled, strip out all query params except for `page`. + if ($this->config('ignore_query_strings')) { + $query = collect(explode('&', $query))->filter(function ($param) { + return Str::startsWith($param, 'page='); + })->implode('&'); + } if ($this->isBasenameTooLong($basename = $slug.'_'.$query.'.html')) { $basename = $slug.'_lqs_'.md5($query).'.html'; From 98ceb6f24d4ac75e6671c362d8a9f9d886caad1b Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Fri, 9 Feb 2024 16:52:19 +0000 Subject: [PATCH 05/18] Make a "whitelisted query parameters" option By default, we'll have the `page` query parameter in here so pagination works. However, if you change the pagination param name or have some other query parameter you want to whitelist, then that setting lets you do that. --- config/static_caching.php | 4 ++++ src/StaticCaching/Cachers/AbstractCacher.php | 15 +++++++++------ src/StaticCaching/Cachers/FileCacher.php | 9 ++++++--- src/StaticCaching/StaticCacheManager.php | 1 + 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/config/static_caching.php b/config/static_caching.php index 0e20114a1b..e379ff26bd 100644 --- a/config/static_caching.php +++ b/config/static_caching.php @@ -98,6 +98,10 @@ 'ignore_query_strings' => false, + 'whitelisted_query_parameters' => [ + 'page', + ], + /* |-------------------------------------------------------------------------- | Replacers diff --git a/src/StaticCaching/Cachers/AbstractCacher.php b/src/StaticCaching/Cachers/AbstractCacher.php index 90e7560175..5cd62a6357 100644 --- a/src/StaticCaching/Cachers/AbstractCacher.php +++ b/src/StaticCaching/Cachers/AbstractCacher.php @@ -139,16 +139,19 @@ public function getUrl(Request $request) { $url = $request->getUri(); - // When ignore_query_strings is enabled, strip out all query params except for `page`. if ($this->config('ignore_query_strings')) { - $urlWithoutParams = Arr::get(explode('?', $url), 0); - $queryParams = Arr::get(explode('?', $url), 1); + $originalUrl = $url; - $url = $urlWithoutParams; + $url = Arr::get(explode('?', $originalUrl), 0); + $queryParams = Arr::get(explode('?', $originalUrl), 1); if ($queryParams) { - $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) { - return Str::startsWith($param, 'page='); + $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) + ->map(fn ($param) => Str::ensureRight($param, '=')) + ->all(); + + $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) use ($whitelistedQueryParams) { + return Str::startsWith($param, $whitelistedQueryParams); })->implode('&'); } } diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index 75604c0b3d..49c046a4f5 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -167,10 +167,13 @@ public function getFilePath($url, $site = null) $slug = $pathParts['basename']; $query = Arr::get($urlParts, 'query', ''); - // When ignore_query_strings is enabled, strip out all query params except for `page`. if ($this->config('ignore_query_strings')) { - $query = collect(explode('&', $query))->filter(function ($param) { - return Str::startsWith($param, 'page='); + $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) + ->map(fn ($param) => Str::ensureRight($param, '=')) + ->all(); + + $query = collect(explode('&', $query))->filter(function ($param) use ($whitelistedQueryParams) { + return Str::startsWith($param, $whitelistedQueryParams); })->implode('&'); } diff --git a/src/StaticCaching/StaticCacheManager.php b/src/StaticCaching/StaticCacheManager.php index d61c50046e..4e591cb6df 100644 --- a/src/StaticCaching/StaticCacheManager.php +++ b/src/StaticCaching/StaticCacheManager.php @@ -54,6 +54,7 @@ protected function getConfig($name) return array_merge($config, [ 'exclude' => $this->app['config']['statamic.static_caching.exclude'] ?? [], 'ignore_query_strings' => $this->app['config']['statamic.static_caching.ignore_query_strings'] ?? false, + 'whitelisted_query_parameters' => $this->app['config']['statamic.static_caching.whitelisted_query_parameters'] ?? [], 'locale' => Site::current()->handle(), ]); } From 58d3d49467c99015ee463c7824595535529f9a04 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Sat, 10 Feb 2024 10:17:06 +0000 Subject: [PATCH 06/18] Add test --- tests/StaticCaching/CacherTest.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/StaticCaching/CacherTest.php b/tests/StaticCaching/CacherTest.php index 56f915481a..7cb2c96875 100644 --- a/tests/StaticCaching/CacherTest.php +++ b/tests/StaticCaching/CacherTest.php @@ -57,6 +57,19 @@ public function gets_a_url_with_query_strings_disabled() $this->assertEquals('http://example.com/test', $cacher->getUrl($request)); } + /** @test */ + public function gets_a_url_with_query_strings_disabled_and_whitelisted_query_params() + { + $cacher = $this->cacher(['ignore_query_strings' => true, 'whitelisted_query_parameters' => ['page']]); + + $request = Request::create('http://example.com/test', 'GET', [ + 'page' => 5, + 'foo' => 'bar', + ]); + + $this->assertEquals('http://example.com/test?page=5', $cacher->getUrl($request)); + } + /** @test */ public function gets_the_base_url_using_the_deprecated_config_value() { From 1231a6a011035329100e028274bdadbb7bf9e9b8 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Sat, 10 Feb 2024 10:17:45 +0000 Subject: [PATCH 07/18] Fix `?` being appended without any whitelisted params present --- src/StaticCaching/Cachers/AbstractCacher.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StaticCaching/Cachers/AbstractCacher.php b/src/StaticCaching/Cachers/AbstractCacher.php index 5cd62a6357..fbe07ed653 100644 --- a/src/StaticCaching/Cachers/AbstractCacher.php +++ b/src/StaticCaching/Cachers/AbstractCacher.php @@ -145,11 +145,11 @@ public function getUrl(Request $request) $url = Arr::get(explode('?', $originalUrl), 0); $queryParams = Arr::get(explode('?', $originalUrl), 1); - if ($queryParams) { - $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) - ->map(fn ($param) => Str::ensureRight($param, '=')) - ->all(); + $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) + ->map(fn ($param) => Str::ensureRight($param, '=')) + ->all(); + if ($queryParams && $whitelistedQueryParams) { $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) use ($whitelistedQueryParams) { return Str::startsWith($param, $whitelistedQueryParams); })->implode('&'); From 4c7d410fb91bfd17ae3b2cdae214d47958aba0f4 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Sat, 10 Feb 2024 11:37:34 +0000 Subject: [PATCH 08/18] Make pagination work with queued warming --- src/Console/Commands/StaticWarmJob.php | 7 +++++- tests/Console/Commands/StaticWarmJobTest.php | 26 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/Console/Commands/StaticWarmJob.php b/src/Console/Commands/StaticWarmJob.php index e9cd038040..8f8af12b22 100644 --- a/src/Console/Commands/StaticWarmJob.php +++ b/src/Console/Commands/StaticWarmJob.php @@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; +use Statamic\Support\Arr; class StaticWarmJob implements ShouldQueue { @@ -24,6 +25,10 @@ public function __construct(Request $request) public function handle(Client $client) { - $client->send($this->request); + $response = $client->send($this->request); + + if ($response->hasHeader('Statamic-Pagination-Next')) { + StaticWarmJob::dispatch(new Request('GET', Arr::first($response->getHeader('Statamic-Pagination-Next')))); + } } } diff --git a/tests/Console/Commands/StaticWarmJobTest.php b/tests/Console/Commands/StaticWarmJobTest.php index df77989eeb..dba0b5760d 100644 --- a/tests/Console/Commands/StaticWarmJobTest.php +++ b/tests/Console/Commands/StaticWarmJobTest.php @@ -7,6 +7,7 @@ use GuzzleHttp\HandlerStack; use GuzzleHttp\Psr7\Request; use GuzzleHttp\Psr7\Response; +use Illuminate\Support\Facades\Queue; use Statamic\Console\Commands\StaticWarmJob; use Tests\TestCase; @@ -29,4 +30,29 @@ public function it_sends_a_get_request() $this->assertEquals('/about', $mock->getLastRequest()->getUri()->getPath()); } + + /** @test */ + public function it_sends_a_get_request_and_dispatches_static_warm_job_for_page_with_pagination() + { + Queue::fake(); + + $mock = new MockHandler([ + (new Response(200))->withHeader('Statamic-Pagination-Next', '/blog?page=2'), + ]); + + $handlerStack = HandlerStack::create($mock); + + $client = new Client(['handler' => $handlerStack]); + + $job = new StaticWarmJob(new Request('GET', '/blog')); + + $job->handle($client); + + $this->assertEquals('/blog', $mock->getLastRequest()->getUri()->getPath()); + + Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) { + return $job->request->getUri()->getPath() === '/blog' + && $job->request->getUri()->getQuery() === 'page=2'; + }); + } } From 4221c7cb1f2567ee11946918ac786a2e1742f0c4 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Sat, 10 Feb 2024 12:26:01 +0000 Subject: [PATCH 09/18] Refactor --- src/Console/Commands/StaticWarm.php | 82 ++++++++++++-------- src/Console/Commands/StaticWarmJob.php | 12 ++- src/StaticCaching/Middleware/Cache.php | 6 +- tests/Console/Commands/StaticWarmJobTest.php | 16 +++- 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/src/Console/Commands/StaticWarm.php b/src/Console/Commands/StaticWarm.php index 947b002a3e..a68f449dff 100644 --- a/src/Console/Commands/StaticWarm.php +++ b/src/Console/Commands/StaticWarm.php @@ -42,8 +42,6 @@ class StaticWarm extends Command private $uris; - private $additionalUris = []; - public function handle() { if (! config('statamic.static_caching.strategy')) { @@ -61,29 +59,7 @@ public function handle() $this->comment('Please wait. This may take a while if you have a lot of content.'); - $this->output->newLine(); - $this->line('Compiling URLs...'); - $this->output->newLine(); - - $client = new Client([ - 'verify' => $this->shouldVerifySsl(), - 'auth' => $this->option('user') && $this->option('password') - ? [$this->option('user'), $this->option('password')] - : null, - ]); - - $this->warm($client, $this->requests()); - - // TODO: Figure out if this works with queued warming.. we may need to figure out a different way if not. - while (count($this->additionalUris) > 0) { - $additionalRequests = collect($this->additionalUris) - ->map(fn ($uri) => new Request('GET', $uri)) - ->all(); - - $this->additionalUris = []; - - $this->warm($client, $additionalRequests); - } + $this->warm(); $this->output->newLine(); $this->info($this->shouldQueue @@ -94,8 +70,15 @@ public function handle() return 0; } - private function warm(Client $client, array $requests): void + private function warm(): void { + $this->output->newLine(); + $this->line('Compiling URLs...'); + + $requests = $this->requests(); + + $this->output->newLine(); + if ($this->shouldQueue) { $queue = config('statamic.static_caching.warm_queue'); $this->line(sprintf('Adding %s requests onto %squeue...', count($requests), $queue ? $queue.' ' : '')); @@ -106,7 +89,7 @@ private function warm(Client $client, array $requests): void } else { $this->line('Visiting '.count($requests).' URLs...'); - $pool = new Pool($client, $requests, [ + $pool = new Pool($this->client(), $requests, [ 'concurrency' => $this->concurrency(), 'fulfilled' => [$this, 'outputSuccessLine'], 'rejected' => [$this, 'outputFailureLine'], @@ -118,6 +101,37 @@ private function warm(Client $client, array $requests): void } } + private function warmPaginatedPages(string $url, int $currentPage, int $totalPages, string $pageName): void + { + $urls = collect(range($currentPage, $totalPages))->map(function ($page) use ($url, $pageName) { + return "{$url}?{$pageName}={$page}"; + }); + + $requests = $urls->map(fn (string $url) => new Request('GET', $url))->all(); + + $pool = new Pool($this->client(), $requests, [ + 'concurrency' => $this->concurrency(), + 'fulfilled' => function (Response $response, $index) use ($urls) { + $this->checkLine($this->getRelativeUri($urls->get($index))); + }, + 'rejected' => [$this, 'outputFailureLine'], + ]); + + $promise = $pool->promise(); + + $promise->wait(); + } + + private function client(): Client + { + return new Client([ + 'verify' => $this->shouldVerifySsl(), + 'auth' => $this->option('user') && $this->option('password') + ? [$this->option('user'), $this->option('password')] + : null, + ]); + } + private function concurrency(): int { $strategy = config('statamic.static_caching.strategy'); @@ -127,16 +141,18 @@ private function concurrency(): int public function outputSuccessLine(Response $response, $index): void { - $this->checkLine($this->getRelativeUri($index)); + $this->checkLine($this->getRelativeUri($this->uris()->get($index))); + + if ($response->hasHeader('X-Statamic-Pagination')) { + [$currentPage, $totalPages, $pageName] = $response->getHeader('X-Statamic-Pagination'); - if ($response->hasHeader('Statamic-Pagination-Next')) { - $this->additionalUris[] = Arr::first($response->getHeader('Statamic-Pagination-Next')); + $this->warmPaginatedPages($this->uris()->get($index), $currentPage, $totalPages, $pageName); } } public function outputFailureLine($exception, $index): void { - $uri = $this->getRelativeUri($index); + $uri = $this->getRelativeUri($this->uris()->get($index)); if ($exception instanceof RequestException && $exception->hasResponse()) { $response = $exception->getResponse(); @@ -153,9 +169,9 @@ public function outputFailureLine($exception, $index): void $this->crossLine("$uri → $message"); } - private function getRelativeUri(int $index): string + private function getRelativeUri(string $uri): string { - return Str::start(Str::after($this->uris()->get($index), config('app.url')), '/'); + return Str::start(Str::after($uri, config('app.url')), '/'); } private function requests() diff --git a/src/Console/Commands/StaticWarmJob.php b/src/Console/Commands/StaticWarmJob.php index 8f8af12b22..ffc1e3bb4f 100644 --- a/src/Console/Commands/StaticWarmJob.php +++ b/src/Console/Commands/StaticWarmJob.php @@ -27,8 +27,16 @@ public function handle(Client $client) { $response = $client->send($this->request); - if ($response->hasHeader('Statamic-Pagination-Next')) { - StaticWarmJob::dispatch(new Request('GET', Arr::first($response->getHeader('Statamic-Pagination-Next')))); + if ($response->hasHeader('X-Statamic-Pagination')) { + [$currentPage, $totalPages, $pageName] = $response->getHeader('X-Statamic-Pagination'); + + collect(range($currentPage, $totalPages)) + ->map(function (int $page) use ($pageName) { + return "{$this->request->getUri()}?{$pageName}={$page}"; + }) + ->each(function (string $uri) { + StaticWarmJob::dispatch(new Request('GET', $uri)); + }); } } } diff --git a/src/StaticCaching/Middleware/Cache.php b/src/StaticCaching/Middleware/Cache.php index 1b4139e56e..70c881fd4c 100644 --- a/src/StaticCaching/Middleware/Cache.php +++ b/src/StaticCaching/Middleware/Cache.php @@ -69,7 +69,11 @@ public function handle($request, Closure $next) if ($paginator = Blink::get('tag-paginator')) { if ($paginator->hasMorePages()) { - $response->headers->set('Statamic-Pagination-Next', $paginator->nextPageUrl()); + $response->headers->set('X-Statamic-Pagination', [ + 'current' => $paginator->currentPage(), + 'total' => $paginator->lastPage(), + 'name' => $paginator->getPageName(), + ]); } } } elseif (! $response->isRedirect()) { diff --git a/tests/Console/Commands/StaticWarmJobTest.php b/tests/Console/Commands/StaticWarmJobTest.php index dba0b5760d..828322b50c 100644 --- a/tests/Console/Commands/StaticWarmJobTest.php +++ b/tests/Console/Commands/StaticWarmJobTest.php @@ -37,7 +37,11 @@ public function it_sends_a_get_request_and_dispatches_static_warm_job_for_page_w Queue::fake(); $mock = new MockHandler([ - (new Response(200))->withHeader('Statamic-Pagination-Next', '/blog?page=2'), + (new Response(200))->withHeader('X-Statamic-Pagination', [ + 'current' => 1, + 'total' => 3, + 'name' => 'page', + ]), ]); $handlerStack = HandlerStack::create($mock); @@ -50,9 +54,19 @@ public function it_sends_a_get_request_and_dispatches_static_warm_job_for_page_w $this->assertEquals('/blog', $mock->getLastRequest()->getUri()->getPath()); + Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) { + return $job->request->getUri()->getPath() === '/blog' + && $job->request->getUri()->getQuery() === 'page=1'; + }); + Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) { return $job->request->getUri()->getPath() === '/blog' && $job->request->getUri()->getQuery() === 'page=2'; }); + + Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) { + return $job->request->getUri()->getPath() === '/blog' + && $job->request->getUri()->getQuery() === 'page=3'; + }); } } From e2373b8d6b363e17668351195d97b7812a1ccfd6 Mon Sep 17 00:00:00 2001 From: duncanmcclean Date: Sat, 10 Feb 2024 12:45:53 +0000 Subject: [PATCH 10/18] Fix styling --- src/Console/Commands/StaticWarm.php | 1 - src/Console/Commands/StaticWarmJob.php | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Console/Commands/StaticWarm.php b/src/Console/Commands/StaticWarm.php index a68f449dff..2818e4e203 100644 --- a/src/Console/Commands/StaticWarm.php +++ b/src/Console/Commands/StaticWarm.php @@ -19,7 +19,6 @@ use Statamic\Facades\URL; use Statamic\Http\Controllers\FrontendController; use Statamic\StaticCaching\Cacher as StaticCacher; -use Statamic\Support\Arr; use Statamic\Support\Str; use Statamic\Taxonomies\LocalizedTerm; use Statamic\Taxonomies\Taxonomy; diff --git a/src/Console/Commands/StaticWarmJob.php b/src/Console/Commands/StaticWarmJob.php index ffc1e3bb4f..ea3aa4d727 100644 --- a/src/Console/Commands/StaticWarmJob.php +++ b/src/Console/Commands/StaticWarmJob.php @@ -8,7 +8,6 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Statamic\Support\Arr; class StaticWarmJob implements ShouldQueue { From 60ba73c74f09ce242d04a5429e7d2cd592301cf4 Mon Sep 17 00:00:00 2001 From: Jason Varga Date: Fri, 22 Mar 2024 13:01:41 -0400 Subject: [PATCH 11/18] avoid controversial word --- config/static_caching.php | 2 +- src/StaticCaching/Cachers/AbstractCacher.php | 8 ++++---- src/StaticCaching/Cachers/FileCacher.php | 6 +++--- src/StaticCaching/StaticCacheManager.php | 2 +- tests/StaticCaching/CacherTest.php | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/static_caching.php b/config/static_caching.php index b05102c4bf..0904f1da99 100644 --- a/config/static_caching.php +++ b/config/static_caching.php @@ -102,7 +102,7 @@ 'ignore_query_strings' => false, - 'whitelisted_query_parameters' => [ + 'allowed_query_parameters' => [ 'page', ], diff --git a/src/StaticCaching/Cachers/AbstractCacher.php b/src/StaticCaching/Cachers/AbstractCacher.php index fbe07ed653..49685516ce 100644 --- a/src/StaticCaching/Cachers/AbstractCacher.php +++ b/src/StaticCaching/Cachers/AbstractCacher.php @@ -145,13 +145,13 @@ public function getUrl(Request $request) $url = Arr::get(explode('?', $originalUrl), 0); $queryParams = Arr::get(explode('?', $originalUrl), 1); - $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) + $allowedQueryParams = collect($this->config('allowed_query_parameters', [])) ->map(fn ($param) => Str::ensureRight($param, '=')) ->all(); - if ($queryParams && $whitelistedQueryParams) { - $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) use ($whitelistedQueryParams) { - return Str::startsWith($param, $whitelistedQueryParams); + if ($queryParams && $allowedQueryParams) { + $url .= '?'.collect(explode('&', $queryParams))->filter(function ($param) use ($allowedQueryParams) { + return Str::startsWith($param, $allowedQueryParams); })->implode('&'); } } diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index 5f0a90f973..1ee2f734cc 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -168,12 +168,12 @@ public function getFilePath($url, $site = null) $query = Arr::get($urlParts, 'query', ''); if ($this->config('ignore_query_strings')) { - $whitelistedQueryParams = collect($this->config('whitelisted_query_parameters', [])) + $allowedQueryParams = collect($this->config('allowed_query_parameters', [])) ->map(fn ($param) => Str::ensureRight($param, '=')) ->all(); - $query = collect(explode('&', $query))->filter(function ($param) use ($whitelistedQueryParams) { - return Str::startsWith($param, $whitelistedQueryParams); + $query = collect(explode('&', $query))->filter(function ($param) use ($allowedQueryParams) { + return Str::startsWith($param, $allowedQueryParams); })->implode('&'); } diff --git a/src/StaticCaching/StaticCacheManager.php b/src/StaticCaching/StaticCacheManager.php index 03d19734c0..7dac6e57e2 100644 --- a/src/StaticCaching/StaticCacheManager.php +++ b/src/StaticCaching/StaticCacheManager.php @@ -57,7 +57,7 @@ protected function getConfig($name) return array_merge($config, [ 'exclude' => $this->app['config']['statamic.static_caching.exclude'] ?? [], 'ignore_query_strings' => $this->app['config']['statamic.static_caching.ignore_query_strings'] ?? false, - 'whitelisted_query_parameters' => $this->app['config']['statamic.static_caching.whitelisted_query_parameters'] ?? [], + 'allowed_query_parameters' => $this->app['config']['statamic.static_caching.allowed_query_parameters'] ?? [], 'locale' => Site::current()->handle(), ]); } diff --git a/tests/StaticCaching/CacherTest.php b/tests/StaticCaching/CacherTest.php index 7cb2c96875..909b24f58a 100644 --- a/tests/StaticCaching/CacherTest.php +++ b/tests/StaticCaching/CacherTest.php @@ -60,7 +60,7 @@ public function gets_a_url_with_query_strings_disabled() /** @test */ public function gets_a_url_with_query_strings_disabled_and_whitelisted_query_params() { - $cacher = $this->cacher(['ignore_query_strings' => true, 'whitelisted_query_parameters' => ['page']]); + $cacher = $this->cacher(['ignore_query_strings' => true, 'allowed_query_parameters' => ['page']]); $request = Request::create('http://example.com/test', 'GET', [ 'page' => 5, From 3ee8e01e9ded812213134e944b0750827bed3280 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Tue, 16 Jul 2024 12:03:20 +0100 Subject: [PATCH 12/18] use `clientConfig()` method when new'ing up client --- src/Console/Commands/StaticWarm.php | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/Console/Commands/StaticWarm.php b/src/Console/Commands/StaticWarm.php index 7fecc41fc7..3aeb320e98 100644 --- a/src/Console/Commands/StaticWarm.php +++ b/src/Console/Commands/StaticWarm.php @@ -124,16 +124,6 @@ private function warmPaginatedPages(string $url, int $currentPage, int $totalPag $promise->wait(); } - private function client(): Client - { - return new Client([ - 'verify' => $this->shouldVerifySsl(), - 'auth' => $this->option('user') && $this->option('password') - ? [$this->option('user'), $this->option('password')] - : null, - ]); - } - private function concurrency(): int { $strategy = config('statamic.static_caching.strategy'); @@ -141,6 +131,11 @@ private function concurrency(): int return config("statamic.static_caching.strategies.$strategy.warm_concurrency", 25); } + private function client(): Client + { + return new Client($this->clientConfig()); + } + private function clientConfig(): array { return [ From 001d8704af2a6de765116b2e7497664031bf85ec Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Tue, 16 Jul 2024 12:03:51 +0100 Subject: [PATCH 13/18] this should be allowed --- tests/StaticCaching/CacherTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/StaticCaching/CacherTest.php b/tests/StaticCaching/CacherTest.php index 515b8f407a..be1b8788df 100644 --- a/tests/StaticCaching/CacherTest.php +++ b/tests/StaticCaching/CacherTest.php @@ -58,7 +58,7 @@ public function gets_a_url_with_query_strings_disabled() } #[Test] - public function gets_a_url_with_query_strings_disabled_and_whitelisted_query_params() + public function gets_a_url_with_query_strings_disabled_and_allowed_query_params() { $cacher = $this->cacher(['ignore_query_strings' => true, 'allowed_query_parameters' => ['page']]); From bb344a5df298c85a152f2abda255f6912517e48a Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Fri, 9 Aug 2024 17:05:20 +0100 Subject: [PATCH 14/18] Pass the Guzzle config through to the StaticWarmJob --- src/Console/Commands/StaticWarmJob.php | 2 +- tests/Console/Commands/StaticWarmJobTest.php | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Console/Commands/StaticWarmJob.php b/src/Console/Commands/StaticWarmJob.php index 6764aa74c5..09de9b93aa 100644 --- a/src/Console/Commands/StaticWarmJob.php +++ b/src/Console/Commands/StaticWarmJob.php @@ -34,7 +34,7 @@ public function handle() return "{$this->request->getUri()}?{$pageName}={$page}"; }) ->each(function (string $uri) { - StaticWarmJob::dispatch(new Request('GET', $uri)); + StaticWarmJob::dispatch(new Request('GET', $uri), $this->clientConfig); }); } } diff --git a/tests/Console/Commands/StaticWarmJobTest.php b/tests/Console/Commands/StaticWarmJobTest.php index 8d538e78b7..8baa3247fd 100644 --- a/tests/Console/Commands/StaticWarmJobTest.php +++ b/tests/Console/Commands/StaticWarmJobTest.php @@ -44,11 +44,9 @@ public function it_sends_a_get_request_and_dispatches_static_warm_job_for_page_w $handlerStack = HandlerStack::create($mock); - $client = new Client(['handler' => $handlerStack]); + $job = new StaticWarmJob(new Request('GET', '/blog'), ['handler' => $handlerStack]); - $job = new StaticWarmJob(new Request('GET', '/blog')); - - $job->handle($client); + $job->handle(); $this->assertEquals('/blog', $mock->getLastRequest()->getUri()->getPath()); From d3c23ddf657a3df9e37cc994364ca18957fb761c Mon Sep 17 00:00:00 2001 From: duncanmcclean Date: Thu, 12 Sep 2024 09:36:23 +0000 Subject: [PATCH 15/18] Fix styling --- src/StaticCaching/Cachers/AbstractCacher.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/StaticCaching/Cachers/AbstractCacher.php b/src/StaticCaching/Cachers/AbstractCacher.php index 2041cafeba..eb125047d2 100644 --- a/src/StaticCaching/Cachers/AbstractCacher.php +++ b/src/StaticCaching/Cachers/AbstractCacher.php @@ -9,7 +9,6 @@ use Statamic\Facades\Site; use Statamic\StaticCaching\Cacher; use Statamic\StaticCaching\UrlExcluder; -use Statamic\Support\Arr; use Statamic\Support\Str; abstract class AbstractCacher implements Cacher From 9991ea49eef5f4f6e6a651965fc4f1b6aebdd129 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Thu, 12 Sep 2024 10:37:01 +0100 Subject: [PATCH 16/18] no longer needed --- src/StaticCaching/Cachers/FileCacher.php | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index efda82e709..7be32839a5 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -167,17 +167,7 @@ public function getFilePath($url, $site = null) $urlParts = parse_url($url); $pathParts = pathinfo($urlParts['path']); $slug = $pathParts['basename']; - $query = Arr::get($urlParts, 'query', ''); - - if ($this->config('ignore_query_strings')) { - $allowedQueryParams = collect($this->config('allowed_query_parameters', [])) - ->map(fn ($param) => Str::ensureRight($param, '=')) - ->all(); - - $query = collect(explode('&', $query))->filter(function ($param) use ($allowedQueryParams) { - return Str::startsWith($param, $allowedQueryParams); - })->implode('&'); - } + $query = $this->config('ignore_query_strings') ? '' : Arr::get($urlParts, 'query', ''); if ($this->isBasenameTooLong($basename = $slug.'_'.$query.'.html')) { $basename = $slug.'_lqs_'.md5($query).'.html'; From 83ee4dac09f87dd24d8e2ea32058fb5650471610 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Thu, 12 Sep 2024 10:41:00 +0100 Subject: [PATCH 17/18] Revert "no longer needed" This reverts commit 9991ea49eef5f4f6e6a651965fc4f1b6aebdd129. --- src/StaticCaching/Cachers/FileCacher.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index 7be32839a5..efda82e709 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -167,7 +167,17 @@ public function getFilePath($url, $site = null) $urlParts = parse_url($url); $pathParts = pathinfo($urlParts['path']); $slug = $pathParts['basename']; - $query = $this->config('ignore_query_strings') ? '' : Arr::get($urlParts, 'query', ''); + $query = Arr::get($urlParts, 'query', ''); + + if ($this->config('ignore_query_strings')) { + $allowedQueryParams = collect($this->config('allowed_query_parameters', [])) + ->map(fn ($param) => Str::ensureRight($param, '=')) + ->all(); + + $query = collect(explode('&', $query))->filter(function ($param) use ($allowedQueryParams) { + return Str::startsWith($param, $allowedQueryParams); + })->implode('&'); + } if ($this->isBasenameTooLong($basename = $slug.'_'.$query.'.html')) { $basename = $slug.'_lqs_'.md5($query).'.html'; From 33ffe1064b747eeb7d132aa2506021c38f8bb640 Mon Sep 17 00:00:00 2001 From: Duncan McClean Date: Thu, 12 Sep 2024 10:47:08 +0100 Subject: [PATCH 18/18] tidy up --- src/StaticCaching/Cachers/FileCacher.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/StaticCaching/Cachers/FileCacher.php b/src/StaticCaching/Cachers/FileCacher.php index efda82e709..8e653d300d 100644 --- a/src/StaticCaching/Cachers/FileCacher.php +++ b/src/StaticCaching/Cachers/FileCacher.php @@ -170,13 +170,13 @@ public function getFilePath($url, $site = null) $query = Arr::get($urlParts, 'query', ''); if ($this->config('ignore_query_strings')) { - $allowedQueryParams = collect($this->config('allowed_query_parameters', [])) + $allowedQueryParams = collect($this->config('allowed_query_strings', [])) ->map(fn ($param) => Str::ensureRight($param, '=')) ->all(); - $query = collect(explode('&', $query))->filter(function ($param) use ($allowedQueryParams) { - return Str::startsWith($param, $allowedQueryParams); - })->implode('&'); + $query = collect(explode('&', $query)) + ->filter(fn ($param) => Str::startsWith($param, $allowedQueryParams)) + ->implode('&'); } if ($this->isBasenameTooLong($basename = $slug.'_'.$query.'.html')) {