PageSpeed Serf fetch failure rate extremely high; only 4 of 12 recent fetches fully successful; is fetching working? #1746
Description
Hi, in I am running an Ubuntu 20.04 LEMP server with the following modules enabled:
ngx_http_brotli_filter_module.so, ngx_http_brotli_static_module.so, ngx_http_cache_purge_module.so ngx_http_modsecurity_module.so, ngx_http_security_headers_module.so, and ngx_pagespeed.so
In my Nginx error logs I am getting many messages similar to the following:
[error] 2924324#2982247: [ngx_pagespeed 1.15.0.0-8811] PageSpeed Serf fetch failure rate extremely high; only 7 of 18 recent fetches fully successful; is fetching working?
Can anyone help me understand what these errors mean and how exactly I can fix them?
Below is my nginx.conf file, my virtual host file, and my pagespeed.conf file.
#nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
#include /etc/nginx/modules-enabled/*.conf;
load_module modules/ngx_http_security_headers_module.so;
load_module modules/ngx_http_modsecurity_module.so;
load_module modules/ngx_pagespeed.so;
load_module modules/ngx_http_cache_purge_module.so;
load_module modules/ngx_http_brotli_filter_module.so;
load_module modules/ngx_http_brotli_static_module.so;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Security Headers
##
security_headers on;
hide_server_tokens on;
##
# Mod Security
##
modsecurity on;
#modsecurity off;
modsecurity_rules_file /etc/nginx/modsec/main.conf;
##
# Basic Settings
##
# Upload settings
client_max_body_size 200M;
client_body_buffer_size 200M;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
types_hash_max_size 4096;
server_tokens off;
server_names_hash_bucket_size 64;
#server_name_in_redirect off; # you might want to comment this out. Do some research.
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# Brotli Compression Settings
##
# The 'brotli off|on' value enables or disbles dynameic or on the fly compression of the content.
brotli on;
# The 'brotli_static on' value enables the Nginx server to check if the pre-compressed files with the .br extensions
# exist or not. The always value allows the server to send pre-compressed content without confirming if the browser
# supports it or not. Since Brotli is resource-intensive, this modules is best suited to reduse the bottleneck situations.
brotli_static on;
# The brotli_comp_level directive sets the dynamic compression quality. It can range from 0 to 11.
brotli_comp_level 6;
#brotli_window 512k;
# Configure a minimum length in order to have the requst compressed, determined by the Content-Length field in the HTTP headers.
#brotli_min_length 21; #20
# Enable dynamic compression for specific MIME types, whereas text/html respnosese are always compressed.
#brotli_types application/atom+xml application/javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/x-javascript application/xhtml+xml application/xml font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
brotli_types application/atom+xml application/javascript application/json application/rss+xml
application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype
application/x-font-ttf application/x-javascript application/xhtml+xml application/xml
font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon
image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml;
##
# Gzip Compression Settings
##
#gzip off;
# Linuxbabe turns gzip_vary on, but we turn it off here because when enabling it, you may have problems clearing the cache.
#gzip_vary off;
#gzip_vary on;
#gzip_proxied any;
#gzip_min_length 1000;
#gzip_comp_level 6;
#gzip_buffers 16 8k;
#gzip_http_version 1.1;
# Linuxbabe (+ added extensions)
#gzip_types application/json application/x-javascript application/javascript application/atom+xml application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml;
##
# Proper Log Format for Netdata
##
# Create a custom Nginx log format called netdata that includes information about request_time, and upstream_response_time, measured in seconds with millisecond resolution.
log_format netdata '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'$request_length $request_time $upstream_response_time '
'"$http_referer" "$http_user_agent"';
##
# WebP Page Cache settings/conditional variables # Use this to enable SERVING webP images in nginx. Dont forget to add a location directive for webP in each virtual configuration files.
##
#map $http_accept $webp_suffix {
#default "";
#"~*webp" ".webp";
#}
##
# SSL Settings
##
ssl_session_cache shared:SSL:10m; #SSL session cache
ssl_session_timeout 1h;
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# FastCGI Cache (by Linuxbabe)
##
# Use this cache path for caching to disk not ram
#fastcgi_cache_path /usr/share/nginx/fastcgi_cache levels=1:2 keys_zone=phpcache:100m max_size=10g inactive=60m use_temp_path=off;
# Use this cache path for caching to ram not disk
fastcgi_cache_path /usr/share/nginx/ramcache/fastcgi_temp/ levels=1:2 keys_zone=phpcache:100m max_size=2g inactive=60m use_temp_path=off;
fastcgi_cache_key "$scheme$request_method$host$request_uri";
# By Easyengine.io
# The line fastcgi_cache_use_stale is what makes caching on Nginx-side unique.
# This line tells Nginx to use old (stale) cached version of page if PHP crashes. This is something not possible with WordPress caching plugins.
fastcgi_cache_use_stale error timeout invalid_header http_500;
fastcgi_read_timeout 300;
# The next 3 lines work with cacheing directly to ram
# https://websiteforstudents.com/improve-nginx-cache-performance-with-tmpfs-on-ubuntu/
#fastcgi_cache_revalidate on;
#fastcgi_cache_background_update on;
#fastcgi_ignore_headers Cache-Control Expires Set-Cookie;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#virtualhost.conf file
server {
listen 80;
listen [::]:80;
server_name mcmo.is www.mcmo.is;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name mcmo.is www.mcmo.is;
root /var/www/mcmo.is/;
###
# SSL (From Mozilla Config Generator: Modern Configuration)
###
# Add Strict Transport Security Response Header with "always Paramater", to help prevent MITM attacks.
# HSTS (ngx_http_headers_module is required) (63072000 seconds)
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always;
## Prevent click jacking by adding an X-Frame-Options header
add_header x-frame-options "SAMEORIGIN" always;
#add_header X-Frame-Options "SAMEORIGIN";
# Add a content security policy header. First line is working original. 2nd line is from linuxbabe combined with working original.
add_header Content-Security-Policy "frame-ancestors 'self';";
#add_header Content-Security-Policy upgrade-insecure-requests "frame-ancestors 'self';";
# Secure MIME Types with X-Content-Type-Options. Below line adds the X-Frame-Options header in Nginx.
add_header X-Content-Type-Options nosniff;
# Enable X-XSS-Protection header in Nginx
add_header X-XSS-Protection "1; mode=block";
add_header Referrer-Policy "strict-origin";
add_header Permissions-Policy "geolocation=(),midi=(),sync-xhr=(),microphone=(),camera=(),magnetometer=(),gyroscope=(),fullscreen=(self),payment=()";
# User recommendation from linuxbabe for fastcgi to keep working
#fastcgi_ignore_headers Cache-Control Expires Set-Cookie; # already in nginx.conf
# Path to signed certificate
ssl_certificate /etc/letsencrypt/live/mcmo.is/fullchain.pem; # Managed by admin
# Path to Intermediate certificates
ssl_certificate_key /etc/letsencrypt/live/mcmo.is/privkey.pem; # Managed by admin
# Verify chain of trust of OCSP response using Root CA and Intermediate certs
ssl_trusted_certificate /etc/letsencrypt/live/mcmo.is/chain.pem; # Managed by admin
# Perfect Forward Secrecy Diffie-Hellman 4098 parameters
ssl_dhparam /etc/ssl/private/dhparams4096.pem; # Managed by admin
# Mozilla Modern Compatibilty SSL configuration with OCSP stapling turned on and strict settings for A+ SSL Security rating
ssl_protocols TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLERequires nginx >= 1.13.0 else use TLSv1.2 # Dropping TLSv1.1 for modern compatability.
ssl_ciphers 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
ssl_ecdh_curve secp384r1; # Requires nginx >= 1.1.0
ssl_prefer_server_ciphers on;
ssl_session_timeout 1d;
ssl_session_cache shared:MozSSL:10m; # about 40000 sessions
ssl_session_tickets off;
# OCSP stapling
ssl_stapling on;
ssl_stapling_verify on;
resolver 8.8.8.8 8.8.4.4;
##
# Harden nginx against DDOS
##
# Pestmeester.nl # Change to 10 to really harden.
##
client_header_timeout 15;
client_body_timeout 15;
keepalive_timeout 15;
send_timeout 15;
# Error & Access Logs
access_log /var/log/nginx/mcmo.is.access.log netdata;
error_log /var/log/nginx/mcmo.is.error.log warn;
# Linuxbabe
index index.php index.html index.htm index.nginx-debian.html;
# Linuxbabe
location / {
try_files $uri $uri/ /index.php;
}
# Non-Linuxbabe
#location / {
#try_files $uri $uri/ /index.php?$args;
#}
# Non-Linuxbabe
location ~ /.well-known {
allow all;
}
# Non-Linuxbabe
location = /favicon.ico {
log_not_found off;
access_log off;
}
# Non-Linuxbabe
location = /robots.txt {
allow all;
log_not_found off;
access_log off;
}
# Non-LinuxBabe (https://tonyteaches.tech/wepb-images-nginx/)
# serve png, jpg, jpeg as webp if available
#location ~* ^.+\.(png|jpe?g)$ {
#add_header Vary Accept;
#try_files $uri$webp_suffix $uri =404;
#}
# Linuxbabe
location ~ ^/wp-json/ {
rewrite ^/wp-json/(.*?)$ /?rest_route=/$1 last;
}
# Linuxbabe
location ~* /wp-sitemap.*\.xml {
try_files $uri $uri/ /index.php$is_args$args;
}
# Linuxbabe
error_page 404 /404.html;
error_page 500 502 503 504 /50x.html;
# Linuxbabe
location = /50x.html {
root /usr/share/nginx/html;
}
##
# Pagespeed
##
# enable pagespeed module on this server block
pagespeed on;
pagespeed Domain https://www.mcmo.is;
pagespeed Domain https://mcmo.is;
# Include UNIVERSAL Settings for Pagespeed Module
include /etc/nginx/pagespeed.conf;
# Non-Linuxbabe
# Block access to xmlrpc.php file
#location ~ /xmlrpc.php$ {
#allow 192.168.30.0/24;
#deny all;
#error_page 403 =444;
#include snippets/fastcgi-php.conf;
#fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
#}
# Non-Linuxbabe
# Only allow access of /admin via internal IP
location ^~ /admin/ {
allow 192.168.30.0/24;
deny all;
error_page 403 =444;
include snippets/fastcgi-php.conf;
fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
}
# Non-Linuxbabe
# Allow local only to wp-login page
#location ~ /wp-login.php {
#allow 192.168.30.0/24;
#deny all;
#error_page 403 =444;
#include snippets/fastcgi-php.conf;
#fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
#}
# Non-Linuxbabe
# Disallow PHP In Upload Folder
location /wp-content/uploads/ {
location ~ \.php$ {
deny all;
}
}
# Non-Linuxbabe
# Security Settings For Better Privacy Deny Hidden Files
location ~ /\. {
deny all;
access_log off;
log_not_found off;
}
# Linuxbabe - disable access to hidden files
location ~ /\.ht {
access_log off;
log_not_found off;
deny all;
}
##
# FastCGI Cache
##
# Cache Purging should be restricted to allowed IP addresses.
# If not set, an attacker may be able to wipe your nginx fastcgi cache using simple GET requests. Linuxbabe (User Comment).
# This (supposedly) enables cache purge but restricts it to your ip address and to your loopback address.
location ~ /purge(/.*) {
allow 127.0.0.1;
allow 149.28.125.6; # Replace with your servers IPv4 address
deny all;
# Enable http-cache-purge module in nginx for above IP addresses
fastcgi_cache_purge phpcache "$scheme$request_method$host$1";
}
##
# START SKIP CACHE RULES: Prevent FastCGI caching for certain things/pages
##
set $skip_cache 0;
# POST requests and urls with a query string should always go to PHP
if ($request_method = POST) {
set $skip_cache 1;
}
if ($query_string != "") {
set $skip_cache 1;
}
# Don't cache uris containing the following segments
if ($request_uri ~* "/wp-admin/|/xmlrpc.php|wp-.*.php|^/feed/*|/tag/.*/feed/*|index.php|/.*sitemap.*\.(xml|xsl)") {
set $skip_cache 1;
}
# Don't use the cache for logged in users or recent commenters
if ($http_cookie ~* "comment_author|wordpress_[a-f0-9]+|wp-postpass|wordpress_no_cache|wordpress_logged_in") {
set $skip_cache 1;
}
# Sometimes I want to test the upstream (PHP-FPM and MariaDB) response time, so I also add the following lines to tell Nginx to bypass the FastCGI cache for my own IP addresses.
# Note: This was not fullly understood from linuxbabe tutorial fastcgi chapter, and therefore was modified by admin. Use carefully.
#if ($remote_addr ~* "108.231.125.253|108.231.125.253") {
# set $skip_cache 1;
#}
# Skip the fastCGI Cache for ATNT Public IP, and the local network.
#if ($remote_addr ~* "108.231.125.253|108.231.125.254|192.168.30..*) {
# set $skip_cache 1;
#}
# THIS IS THE MODIFIED CODE REFERENCED in 2 sections above from here.
# Lets deactivate it for now so that we don't bypass FastCGI Cache for our own IP, or fuck things up with modified lines.
#if ($remote_addr ~* "108.231.125.253|192.168.30..*") {
# set $skip_cache 1;
#}
# IMPORTANT NOTE WITH SEO:
# If you use the Yoast SEO or Google XML Sitemap plugins to generate sitemap, then
# you also need to move the Yoast/Google XML rewrite rules below the skip cache rules.
##
# END SKIP CACHE RULES: Prevent FastCGI caching for certain things/pages
##
# Linuxbabe
# Pass Fastcgi to php
location ~ \.php$ {
fastcgi_pass unix:/run/php/php7.4-fpm.sock;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
include fastcgi_params;
include snippets/fastcgi-php.conf;
# Linuxbabe - FastCGI
fastcgi_cache phpcache;
fastcgi_cache_valid 200 301 302 60m;
fastcgi_cache_use_stale error timeout updating invalid_header http_500 http_503;
fastcgi_cache_min_uses 1;
fastcgi_cache_lock on;
# Tell Nginx to send request to upstream PHP-FPM server, instead of trying to find files in the cache.
fastcgi_cache_bypass $skip_cache;
# Tell Nginx not to cache the response.
fastcgi_no_cache $skip_cache;
add_header X-FastCGI-Cache $upstream_cache_status;
add_header X-Content-Type-Options nosniff;
}
# Linuxbabe (Mostly, added some files extensions)
# A long browser cache lifetime can speed up repeat visits to your page
location ~* \.(jpg|ogg|ogv|svgz|eot|otf|mp4|rss|atom|zip|tgz|gz|rar|bz2|doc|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|jpeg|gif|png|webp|svg|woff|woff2|ttf|css|js|ico|xml)$ {
access_log off;
log_not_found off;
expires 360d;
}
}
#pagespeed.conf file
##
# Pagespeed Module
##
# Linuxbabe
pagespeed Disallow "/wp-admin*"; # Boo's edit
pagespeed Disallow "/wp-login.php"; # Boo's edit
pagespeed Disallow "/admin/*"; # Boo's edit
pagespeed FetchHttps enable;
pagespeed NoTransformOptimizedImages on;
pagespeed HttpCacheCompressionLevel 9;
#admin console
pagespeed Statistics on;
pagespeed StatisticsLogging on;
pagespeed LogDir /var/log/pagespeed;
pagespeed AdminPath /pagespeed_admin;
location ~ ^/pagespeed_admin {
allow 127.0.0.1;
allow 109.277.268.139; # Dans IP Address
deny all;
}
# Where to cache files.
# Needs to exist and be writable by nginx. Use tmpfs for best performance.
pagespeed FileCachePath /usr/share/nginx/ramcache/ngx_pagespeed_temp;
# Ensure requests for pagespeed optimized resources go to the pagespeed handler
# and no extraneous headers get set.
location ~ "\.pagespeed\.([a-z]\.)?[a-z]{2}\.[^.]{10}\.[^.]+" {
add_header "" "";
}
location ~ "^/pagespeed_static/" { }
location ~ "^/ngx_pagespeed_beacon$" { }
###
# Pagespeed Filters
###
# A good way to reduce CPU load is to change the default RewriteLevel to
# PassThrough, which disables all PageSpeed filters.
# Then you can enable the filter you need.
pagespeed RewriteLevel CoreFilters;
#pagespeed RewriteLevel PassThrough;
#pagespeed RewriteLevel OptimizeForBandwidth;
## Text / HTML
#pagespeed EnableFilters add_head; # Core Filter
#pagespeed EnableFilters remove_quotes; # Core Filter
#pagespeed EnableFilters combine_heads;
pagespeed EnableFilters insert_dns_prefetch; # Extras # Linuxbabe Recommendation
pagespeed EnableFilters collapse_whitespace; # Extras # Linuxbabe Recommendation
#pagespeed EnableFilters convert_meta_tags; # Core Filter
#pagespeed EnableFilters elide_attributes;
#pagespeed EnableFilters pedantic;
#pagespeed EnableFilters remove_comments;
#pagespeed EnableFilters remove_quotes;
pagespeed EnableFilters trim_urls;
## JavaScript
#This filter is likely to cause problems for loading your web pages.
pagespeed EnableFilters defer_javascript; # Extras # Linuxbabe Recommendation
#pagespeed EnableFilters combine_javascript; # Core Filter
#pagespeed EnableFilters canonicalize_javascript_libraries;
#pagespeed EnableFilters inline_javascript; # Core Filter
pagespeed EnableFilters rewrite_javascript; # Extras # Core Filter according to linuxbabe
## CSS
#pagespeed EnableFilters outline_css;
#pagespeed EnableFilters flatten_css_imports; # Core Filter
#pagespeed EnableFilters combine_css; # Core Filter
#pagespeed EnableFilters inline_import_to_link; # Core Filter
#pagespeed EnableFilters inline_css; # Core Filter
pagespeed EnableFilters inline_google_font_css;
#pagespeed EnableFilters move_css_above_scripts;
#pagespeed EnableFilters move_css_to_head;
pagespeed EnableFilters prioritize_critical_css; # Extras # Linuxbabe Recommendation
#pagespeed EnableFilters rewrite_css; # Core Filter
#pagespeed EnableFilters fallback_rewrite_css_urls; # Core Filter
#pagespeed EnableFilters rewrite_style_attributes_with_url; # Core Filter
## Images
pagespeed EnableFilters recompress_images;
pagespeed EnableFilters rewrite_images; # Core Filter # Filter group
pagespeed EnableFilters responsive_images; # Extras
pagespeed EnableFilters dedup_inlined_images; # Extras
pagespeed EnableFilters inline_preview_images; # Extras
pagespeed EnableFilters resize_mobile_images; # Extras
#pagespeed EnableFilters lazyload_images; # Extras # Turn off with wordpress
pagespeed EnableFilters inline_images;
#pagespeed EnableFilters convert_gif_to_png;
pagespeed EnableFilters convert_jpeg_to_progressive;
pagespeed EnableFilters recompress_jpeg;
pagespeed EnableFilters recompress_png;
pagespeed EnableFilters recompress_webp;
#pagespeed EnableFilters strip_image_color_profile;
pagespeed EnableFilters strip_image_meta_data;
pagespeed EnableFilters jpeg_subsampling;
pagespeed EnableFilters convert_png_to_jpeg;
pagespeed EnableFilters resize_images; # Extras #Linuxbabe recommendation on passthrough
#pagespeed EnableFilters resize_rendered_image_dimensions;
pagespeed EnableFilters convert_jpeg_to_webp; # Linuxbabe recommendation on passthrough
pagespeed EnableFilters convert_to_webp_lossless; # Linuxbabe recommendation on passthrough
pagespeed EnableFilters insert_image_dimensions;
pagespeed NoTransformOptimizedImages on;
#pagespeed EnableFilters sprite_images; # Extras
## Other
pagespeed EnableFilters hint_preload_subresources;
#pagespeed EnableFilters extend_cache; # Core Filter
pagespeed EnableFilters in_place_optimize_for_browser;
If anyone can take a look at these files and tell me exactly what the issue is and how I can fix pagespeed, it would be greatly appreciated! Thank you so much!
Dan