Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[5.x] Warm paginated pages with static:warm command #9493

Draft
wants to merge 25 commits into
base: 5.x
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
3c93cf3
When there's more pages, warm them
duncanmcclean Feb 2, 2024
31a699a
Merge remote-tracking branch 'origin/4.x' into warm-paginated-pages
duncanmcclean Feb 8, 2024
0bc87ea
wip
duncanmcclean Feb 8, 2024
d2fdfd7
make things work another way
duncanmcclean Feb 8, 2024
a697658
When `ignore_query_strings` is enabled, keep `page` parameter
duncanmcclean Feb 9, 2024
98ceb6f
Make a "whitelisted query parameters" option
duncanmcclean Feb 9, 2024
58d3d49
Add test
duncanmcclean Feb 10, 2024
1231a6a
Fix `?` being appended without any whitelisted params present
duncanmcclean Feb 10, 2024
4c7d410
Make pagination work with queued warming
duncanmcclean Feb 10, 2024
4221c7c
Refactor
duncanmcclean Feb 10, 2024
e2373b8
Fix styling
duncanmcclean Feb 10, 2024
f369ea6
Merge branch '4.x' into warm-paginated-pages
jasonvarga Mar 22, 2024
60ba73c
avoid controversial word
jasonvarga Mar 22, 2024
dfdb76e
Merge remote-tracking branch 'origin/5.x' into warm-paginated-pages
duncanmcclean May 13, 2024
8aa2997
Merge branch '5.x' into warm-paginated-pages
duncanmcclean Jul 16, 2024
3ee8e01
use `clientConfig()` method when new'ing up client
duncanmcclean Jul 16, 2024
001d870
this should be allowed
duncanmcclean Jul 16, 2024
529861f
Merge branch '5.x' into warm-paginated-pages
duncanmcclean Aug 9, 2024
bb344a5
Pass the Guzzle config through to the StaticWarmJob
duncanmcclean Aug 9, 2024
f59ed01
Merge remote-tracking branch 'origin/5.x' into warm-paginated-pages
duncanmcclean Aug 26, 2024
dc0a1fa
Merge remote-tracking branch 'origin/5.x' into warm-paginated-pages
duncanmcclean Sep 12, 2024
d3c23dd
Fix styling
duncanmcclean Sep 12, 2024
9991ea4
no longer needed
duncanmcclean Sep 12, 2024
83ee4da
Revert "no longer needed"
duncanmcclean Sep 12, 2024
33ffe10
tidy up
duncanmcclean Sep 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 37 additions & 7 deletions src/Console/Commands/StaticWarm.php
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,6 @@ public function handle()

private function warm(): void
{
$client = new Client($this->clientConfig());

$this->output->newLine();
$this->line('Compiling URLs...');

Expand All @@ -97,7 +95,7 @@ private function warm(): void
} else {
$this->line('Visiting '.count($requests).' URLs...');

$pool = new Pool($client, $requests, [
$pool = new Pool($this->client(), $requests, [
'concurrency' => $this->concurrency(),
'fulfilled' => [$this, 'outputSuccessLine'],
'rejected' => [$this, 'outputFailureLine'],
Expand All @@ -109,13 +107,39 @@ private function warm(): void
}
}

private function warmPaginatedPages(string $url, int $currentPage, int $totalPages, string $pageName): void
{
$urls = collect(range($currentPage, $totalPages))->map(function ($page) use ($url, $pageName) {
return "{$url}?{$pageName}={$page}";
});

$requests = $urls->map(fn (string $url) => new Request('GET', $url))->all();

$pool = new Pool($this->client(), $requests, [
'concurrency' => $this->concurrency(),
'fulfilled' => function (Response $response, $index) use ($urls) {
$this->components->twoColumnDetail($this->getRelativeUri($urls->get($index)), '<info>✓ Cached</info>');
},
'rejected' => [$this, 'outputFailureLine'],
]);

$promise = $pool->promise();

$promise->wait();
}

private function concurrency(): int
{
$strategy = config('statamic.static_caching.strategy');

return config("statamic.static_caching.strategies.$strategy.warm_concurrency", 25);
}

private function client(): Client
{
return new Client($this->clientConfig());
}

private function clientConfig(): array
{
return [
Expand All @@ -128,12 +152,18 @@ private function clientConfig(): array

public function outputSuccessLine(Response $response, $index): void
{
$this->components->twoColumnDetail($this->getRelativeUri($index), '<info>✓ Cached</info>');
$this->components->twoColumnDetail($this->getRelativeUri($this->uris()->get($index)), '<info>✓ Cached</info>');

if ($response->hasHeader('X-Statamic-Pagination')) {
[$currentPage, $totalPages, $pageName] = $response->getHeader('X-Statamic-Pagination');

$this->warmPaginatedPages($this->uris()->get($index), $currentPage, $totalPages, $pageName);
}
}

public function outputFailureLine($exception, $index): void
{
$uri = $this->getRelativeUri($index);
$uri = $this->getRelativeUri($this->uris()->get($index));

if ($exception instanceof RequestException && $exception->hasResponse()) {
$response = $exception->getResponse();
Expand All @@ -150,9 +180,9 @@ public function outputFailureLine($exception, $index): void
$this->components->twoColumnDetail($uri, "<fg=cyan>$message</fg=cyan>");
}

private function getRelativeUri(int $index): string
private function getRelativeUri(string $uri): string
{
return Str::start(Str::after($this->uris()->get($index), config('app.url')), '/');
return Str::start(Str::after($uri, config('app.url')), '/');
}

private function requests()
Expand Down
14 changes: 13 additions & 1 deletion src/Console/Commands/StaticWarmJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@ public function __construct(public Request $request, public array $clientConfig)

public function handle()
{
(new Client($this->clientConfig))->send($this->request);
$response = (new Client($this->clientConfig))->send($this->request);

if ($response->hasHeader('X-Statamic-Pagination')) {
[$currentPage, $totalPages, $pageName] = $response->getHeader('X-Statamic-Pagination');

collect(range($currentPage, $totalPages))
->map(function (int $page) use ($pageName) {
return "{$this->request->getUri()}?{$pageName}={$page}";
})
->each(function (string $uri) {
StaticWarmJob::dispatch(new Request('GET', $uri), $this->clientConfig);
});
}
}
}
12 changes: 11 additions & 1 deletion src/StaticCaching/Cachers/FileCacher.php
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,17 @@ public function getFilePath($url, $site = null)
$urlParts = parse_url($url);
$pathParts = pathinfo($urlParts['path']);
$slug = $pathParts['basename'];
$query = $this->config('ignore_query_strings') ? '' : Arr::get($urlParts, 'query', '');
$query = Arr::get($urlParts, 'query', '');

if ($this->config('ignore_query_strings')) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When we revisit this PR, we'll need to spend some time getting it working again when ignore_query_strings is true, as it looks like our changes in #10701 broke this. 😄

Before #10701, this method ensured that only query parameters configured in the allowed_query_strings array (like page) were included in the filename.

However, it now it seems like the $url being passed into this method already has the query parameters stripped off.

$allowedQueryParams = collect($this->config('allowed_query_strings', []))
->map(fn ($param) => Str::ensureRight($param, '='))
->all();

$query = collect(explode('&', $query))
->filter(fn ($param) => Str::startsWith($param, $allowedQueryParams))
->implode('&');
}

if ($this->isBasenameTooLong($basename = $slug.'_'.$query.'.html')) {
$basename = $slug.'_lqs_'.md5($query).'.html';
Expand Down
11 changes: 11 additions & 0 deletions src/StaticCaching/Middleware/Cache.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache as AppCache;
use Illuminate\Support\Facades\Log;
use Statamic\Facades\Blink;
use Statamic\Facades\StaticCache;
use Statamic\Statamic;
use Statamic\StaticCaching\Cacher;
Expand Down Expand Up @@ -75,6 +76,16 @@ private function handleRequest($request, Closure $next)
$this->makeReplacementsAndCacheResponse($request, $response);

$this->nocache->write();

if ($paginator = Blink::get('tag-paginator')) {
if ($paginator->hasMorePages()) {
$response->headers->set('X-Statamic-Pagination', [
'current' => $paginator->currentPage(),
'total' => $paginator->lastPage(),
'name' => $paginator->getPageName(),
]);
}
}
} elseif (! $response->isRedirect()) {
$this->makeReplacements($response);
}
Expand Down
38 changes: 38 additions & 0 deletions tests/Console/Commands/StaticWarmJobTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use GuzzleHttp\HandlerStack;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
use Illuminate\Support\Facades\Queue;
use PHPUnit\Framework\Attributes\Test;
use Statamic\Console\Commands\StaticWarmJob;
use Tests\TestCase;
Expand All @@ -27,4 +28,41 @@ public function it_sends_a_get_request()

$this->assertEquals('/about', $mock->getLastRequest()->getUri()->getPath());
}

#[Test]
public function it_sends_a_get_request_and_dispatches_static_warm_job_for_page_with_pagination()
{
Queue::fake();

$mock = new MockHandler([
(new Response(200))->withHeader('X-Statamic-Pagination', [
'current' => 1,
'total' => 3,
'name' => 'page',
]),
]);

$handlerStack = HandlerStack::create($mock);

$job = new StaticWarmJob(new Request('GET', '/blog'), ['handler' => $handlerStack]);

$job->handle();

$this->assertEquals('/blog', $mock->getLastRequest()->getUri()->getPath());

Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) {
return $job->request->getUri()->getPath() === '/blog'
&& $job->request->getUri()->getQuery() === 'page=1';
});

Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) {
return $job->request->getUri()->getPath() === '/blog'
&& $job->request->getUri()->getQuery() === 'page=2';
});

Queue::assertPushed(StaticWarmJob::class, function (StaticWarmJob $job) {
return $job->request->getUri()->getPath() === '/blog'
&& $job->request->getUri()->getQuery() === 'page=3';
});
}
}
Loading