From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Aug 2023 16:08:02 -0700 Subject: [PATCH 01/16] Add support for using HTTP proxies --- config/config.example.yml | 11 +++++++++++ shard.lock | 10 +++++++--- shard.yml | 3 +++ src/invidious.cr | 1 + src/invidious/config.cr | 11 +++++++++++ src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 38085a20b..9fdc4edab 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -160,6 +160,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index efb60a59c..85928c47d 100644 --- a/shard.lock +++ b/shard.lock @@ -6,11 +6,11 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.1 + version: 0.1.3 backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.1 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index be06a7df5..ddd510d43 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.1 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index e0bd01015..d41143860 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b8..e12054d0c 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -54,6 +54,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -126,6 +135,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0ed..0e4d8aff3 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,14 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn rescue ex conn.close conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -46,6 +48,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + return client end @@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 4 Oct 2023 14:36:04 -0400 Subject: [PATCH 02/16] Automatically initialize proxy via stdlib override --- .../helpers/crystal_class_overrides.cr | 34 +++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 7 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d8264..71038703b 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.10.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0e4d8aff3..f34f48c57 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,13 +26,15 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -48,7 +50,6 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - return client end From ccb2a6c58ef9b5b7e1c47938cfa4ec574a8560c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 28 Apr 2024 21:34:05 -0700 Subject: [PATCH 03/16] Bump http_proxy to v0.10.3 --- shard.lock | 4 ++-- shard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index 85928c47d..99b4e99c0 100644 --- a/shard.lock +++ b/shard.lock @@ -6,7 +6,7 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.3 + version: 0.1.1 backtracer: git: https://github.com/sija/backtracer.cr.git @@ -22,7 +22,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.10.1 + version: 0.10.3 kemal: git: https://github.com/kemalcr/kemal.git diff --git a/shard.yml b/shard.yml index ddd510d43..70ebad0aa 100644 --- a/shard.yml +++ b/shard.yml @@ -30,7 +30,7 @@ dependencies: version: ~> 0.1.1 http_proxy: github: mamantoha/http_proxy - version: ~> 0.10.1 + version: ~> 0.10.3 development_dependencies: spectator: From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:10:46 -0700 Subject: [PATCH 04/16] Validate override for crystal 1.12.1 --- src/invidious/helpers/crystal_class_overrides.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index 71038703b..a7d2a5e68 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -20,7 +20,7 @@ class HTTP::Client # Override stdlib to automatically initialize proxy if configured # - # Accurate as of crystal 1.10.1 + # Accurate as of crystal 1.12.1 def initialize(@host : String, port = nil, tls : TLSContext = nil) check_host_only(@host) From 288e1dccda2256a9014364d693b3eb3d7933b242 Mon Sep 17 00:00:00 2001 From: giacomocerquone Date: Thu, 13 Jun 2024 01:10:35 +0200 Subject: [PATCH 05/16] Fix player menus hiding onHover --- assets/css/player.css | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/css/player.css b/assets/css/player.css index 50c7a7487..9cb400ad9 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -68,6 +68,7 @@ .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu { margin-bottom: 2em; + padding-top: 2em } .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px; From 480e073fa9be184b6839619c38795af582247c19 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:20:17 -0800 Subject: [PATCH 06/16] Use HTTP pools for image requests to YouTube --- src/invidious.cr | 8 ++++++++ src/invidious/routes/images.cr | 12 +++++------- src/invidious/yt_backend/connection_pool.cr | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e3..81db2c6c5 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -92,6 +92,14 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +YTIMG_POOLS = {} of String => YoutubeConnectionPool + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e1103..1964d597f 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -32,7 +32,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + GGPHT_POOL.client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -80,7 +80,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + get_ytimg_pool(authority).client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -119,7 +119,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + get_ytimg_pool("i9").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -165,8 +165,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -199,8 +198,7 @@ module Invidious::Routes::Images } begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + get_ytimg_pool("i").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca6120831..26bf2773d 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -77,3 +77,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end From 52bc9aa328e44ff32bb1d7f2e05625e4080459c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:42:40 -0800 Subject: [PATCH 07/16] Refactor duplicate logic in image routes --- src/invidious/routes/images.cr | 95 +++++++--------------------------- 1 file changed, 20 insertions(+), 75 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 1964d597f..7fdd33b0f 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin GGPHT_POOL.client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool(authority).client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +63,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool("i9").client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -180,28 +125,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end + begin + get_ytimg_pool("i").client &.get(url) do |resp| + return self.proxy_image(env, resp) end + rescue ex + end + end - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value end + end - proxy_file(response, env) - } + env.response.headers["Access-Control-Allow-Origin"] = "*" - begin - get_ytimg_pool("i").client &.get(url) do |resp| - return request_proc.call(resp) - end - rescue ex + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") end + + return proxy_file(response, env) end end From 06e1a508e8dc5417a61a02ad1eb08e94fb24ae99 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:52:11 -0800 Subject: [PATCH 08/16] Fix headers not being added in image requests Regression from #2364 --- src/invidious/routes/images.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 7fdd33b0f..c41977460 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Images end begin - GGPHT_POOL.client &.get(url) do |resp| + GGPHT_POOL.client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -42,7 +42,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool(authority).client &.get(url) do |resp| + get_ytimg_pool(authority).client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -64,7 +64,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i9").client &.get(url) do |resp| + get_ytimg_pool("i9").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -110,7 +110,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -126,7 +126,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i").client &.get(url) do |resp| + get_ytimg_pool("i").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex From 4bc77b81bf994336e324d84ab82a362b330c827d Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 23 Dec 2023 13:47:47 -0800 Subject: [PATCH 09/16] Move YTIMG_POOLS to connection_pool.cr --- src/invidious.cr | 4 --- src/invidious/yt_backend/connection_pool.cr | 32 ++++++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 81db2c6c5..e0e72415b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -96,10 +96,6 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) -# Mapping of subdomain => YoutubeConnectionPool -# This is needed as we may need to access arbitrary subdomains of ytimg -YTIMG_POOLS = {} of String => YoutubeConnectionPool - # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 26bf2773d..646d0d1ae 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -54,6 +43,21 @@ struct YoutubeConnectionPool end end +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTP::Client.new(url) From 003c6f81dcf6399d1fa808866d0806b915a713ee Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 8 Jan 2024 14:13:38 -0800 Subject: [PATCH 10/16] Preserve connection close header of get_storyboard --- src/invidious/routes/images.cr | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index c41977460..251258ec3 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,9 +41,14 @@ module Invidious::Routes::Images end end + # A callable proc to be used inside #proxy_image + callable_proc = ->(env : HTTP::Server::Context) { + env.response.headers["Connection"] = "close" + } + begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp) + return self.proxy_image(env, resp, callable_proc: callable_proc) end rescue ex end @@ -133,7 +138,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response) + private def self.proxy_image(env, response, callable_proc = nil) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -143,6 +148,10 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" + if callable_proc + callable_proc.call(env) + end + if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 75b68618ab14a9f884ee7215a467bc510e8bd2c2 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 25 Apr 2024 13:28:58 -0700 Subject: [PATCH 11/16] Remove useless proc usage in images.cr --- src/invidious/routes/images.cr | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 251258ec3..639697db5 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,14 +41,10 @@ module Invidious::Routes::Images end end - # A callable proc to be used inside #proxy_image - callable_proc = ->(env : HTTP::Server::Context) { - env.response.headers["Connection"] = "close" - } - begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp, callable_proc: callable_proc) + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -138,7 +134,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response, callable_proc = nil) + private def self.proxy_image(env, response) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -148,10 +144,6 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" - if callable_proc - callable_proc.call(env) - end - if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 84e4746265d6077b1537b626c9742498f9cb253c Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 18 Sep 2024 18:14:28 -0300 Subject: [PATCH 12/16] SigHelper: Reconnect to signature helper Signed-off-by: Fijxu --- src/invidious/helpers/sig_helper.cr | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7d..6d198a427 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end From f51a3b8d2b52f83057d0b3be5686149984e66ada Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 9 Oct 2024 18:07:04 +0200 Subject: [PATCH 13/16] Makefile: Add MT option to enable the 'preview_mt' flag --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 9eb195df8..ec22a0ded 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,11 @@ STATIC := 0 NO_DBG_SYMBOLS := 0 +# Enable multi-threading. +# Warning: Experimental feature!! +# invidious is not stable when MT is enabled. +MT := 0 + FLAGS ?= @@ -19,6 +24,10 @@ ifeq ($(STATIC), 1) FLAGS += --static endif +ifeq ($(MT), 1) + FLAGS += -Dpreview_mt +endif + ifeq ($(NO_DBG_SYMBOLS), 1) FLAGS += --no-debug From 952b3625a0a8fb21ab04bc267f94a21c331109f6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 10 Oct 2024 20:31:22 +0200 Subject: [PATCH 14/16] Add "Filipino (auto-generated)" to the list of caption languages --- locales/en-US.json | 1 + src/invidious/videos/caption.cr | 1 + 2 files changed, 2 insertions(+) diff --git a/locales/en-US.json b/locales/en-US.json index 7827d9c63..c23f6bc33 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -286,6 +286,7 @@ "Esperanto": "Esperanto", "Estonian": "Estonian", "Filipino": "Filipino", + "Filipino (auto-generated)": "Filipino (auto-generated)", "Finnish": "Finnish", "French": "French", "French (auto-generated)": "French (auto-generated)", diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2b..c811cfe1b 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", From d8b893e9ad456598a3e127cec578dd3355141578 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:33:38 +0000 Subject: [PATCH 15/16] Bump CI matrix (#5015) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bfa501d5..411ec7696 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,10 +38,11 @@ jobs: matrix: stable: [true] crystal: - - 1.9.2 - 1.10.1 - 1.11.2 - 1.12.1 + - 1.13.2 + - 1.14.0 include: - crystal: nightly stable: false From 2e3a7ad044b3e37d15d0c87bb33cb85d2d04424f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 30 Oct 2024 17:13:00 +0100 Subject: [PATCH 16/16] Update CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 159916684..f9892e177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ ### Full list of pull requests merged since the last release (newest first) +* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox) +* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox) +* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu) +* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone) +* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite) +* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite) * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox) * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite) * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov) @@ -22,7 +28,10 @@ [#4122]: https://github.com/iv-org/invidious/pull/4122 [#4193]: https://github.com/iv-org/invidious/pull/4193 +[#4270]: https://github.com/iv-org/invidious/pull/4270 +[#4326]: https://github.com/iv-org/invidious/pull/4326 [#4652]: https://github.com/iv-org/invidious/pull/4652 +[#4750]: https://github.com/iv-org/invidious/pull/4750 [#4850]: https://github.com/iv-org/invidious/pull/4850 [#4862]: https://github.com/iv-org/invidious/pull/4862 [#4863]: https://github.com/iv-org/invidious/pull/4863 @@ -33,6 +42,9 @@ [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 [#4942]: https://github.com/iv-org/invidious/pull/4942 +[#4991]: https://github.com/iv-org/invidious/pull/4991 +[#4993]: https://github.com/iv-org/invidious/pull/4993 +[#4995]: https://github.com/iv-org/invidious/pull/4995 ## v2.20240825.2 (2024-08-26)